From 5750fd5f9645ca7f99f8832d76825acceee389dd Mon Sep 17 00:00:00 2001 From: Marco Malavolti <marco.malavolti@gmail.com> Date: Sat, 9 Oct 2021 16:00:13 +0200 Subject: [PATCH] Replaced ECCS2 with ECCS --- .gitignore | 14 +- .gitlab/issue_templates/bug.md | 2 +- .gitlab/issue_templates/feature_request.md | 2 +- README.md | 188 ++++++++++----------- api.py | 28 +-- clean7daysOldFiles.sh | 2 +- cleanAndRunEccs2.sh => cleanAndRunEccs.sh | 6 +- eccs2-centos.conf => eccs-centos.conf | 10 +- eccs-debian.conf | 17 ++ eccs2wsgi.py => eccs-wsgi.py | 2 +- eccs2.ini.template => eccs.ini.template | 12 +- eccs2.py => eccs.py | 20 +-- eccs.service.template | 18 ++ eccs2-debian.conf | 17 -- eccs2.service.template | 18 -- eccs2properties.py => eccs_properties.py | 64 +++---- retryFailedChecks.py | 24 +-- runEccs2.py => runEccs.py | 36 ++-- utils.py | 52 +++--- web/details_close_transparent.png.1 | Bin 570 -> 0 bytes web/{eccs2.css => eccs.css} | 0 web/{eccs2.js => eccs.js} | 20 +-- web/index.php | 8 +- 23 files changed, 280 insertions(+), 280 deletions(-) rename cleanAndRunEccs2.sh => cleanAndRunEccs.sh (69%) rename eccs2-centos.conf => eccs-centos.conf (50%) create mode 100644 eccs-debian.conf rename eccs2wsgi.py => eccs-wsgi.py (77%) rename eccs2.ini.template => eccs.ini.template (73%) rename eccs2.py => eccs.py (90%) create mode 100644 eccs.service.template delete mode 100644 eccs2-debian.conf delete mode 100644 eccs2.service.template rename eccs2properties.py => eccs_properties.py (69%) rename runEccs2.py => runEccs.py (79%) delete mode 100644 web/details_close_transparent.png.1 rename web/{eccs2.css => eccs.css} (100%) rename web/{eccs2.js => eccs.js} (96%) diff --git a/.gitignore b/.gitignore index 5978a7a..2949413 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,12 @@ *__pycache__* *.pyc -*eccs2venv* -eccs2.pid +*eccs-venv* +eccs.pid chromedriver python* -eccs2.ini -eccs2.conf -eccs2.service -eccs2properties.py +eccs.ini +eccs.conf +eccs.service +eccs_properties.py clean7daysOldFiles.sh -cleanAndRunEccs2.sh +cleanAndRunEccs.sh diff --git a/.gitlab/issue_templates/bug.md b/.gitlab/issue_templates/bug.md index 12c510b..2d72dcd 100644 --- a/.gitlab/issue_templates/bug.md +++ b/.gitlab/issue_templates/bug.md @@ -3,7 +3,7 @@ Please read this! Before opening a new issue, make sure to search for keywords in the issues filtered by the "bug" label: -- https://gitlab.geant.org/marco.malavolti/eccs2/-/issues?label_name%5B%5D=bug +- https://gitlab.geant.org/edugain/eccs/-/issues?label_name%5B%5D=bug and verify that the issue you're about to submit isn't a duplicate. ---> diff --git a/.gitlab/issue_templates/feature_request.md b/.gitlab/issue_templates/feature_request.md index 92a87e5..a3dcab4 100644 --- a/.gitlab/issue_templates/feature_request.md +++ b/.gitlab/issue_templates/feature_request.md @@ -3,7 +3,7 @@ Please read this! Before opening a new issue, make sure to search for keywords in the issues filtered by the "feature_request" label: -- https://gitlab.geant.org/marco.malavolti/eccs2/-/issues?label_name%5B%5D=feature_request +- https://gitlab.geant.org/edugain/eccs/-/issues?label_name%5B%5D=feature_request and verify that the issue you're about to submit isn't a duplicate. ---> diff --git a/README.md b/README.md index 6fcefba..7fc60ab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# EduGAIN Connectivity Check Service 2 - ECCS2 +# EduGAIN Connectivity Check Service - ECCS 1. [Introduction](#introduction) 2. [Check Performed on the IdPs](#check-performed-on-the-idps) @@ -8,25 +8,25 @@ 6. [Requirements Hardware](#requirements-hardware) 7. [Requirements Software](#requirements-software) 8. [HOWTO Install and Configure](#howto-install-and-configure) - * [Install Python 3.9.x](#install-python-39x) + * [Python 3](#python-3) + [CentOS 7 requirements](#centos-7-requirements) + [Debian requirements](#debian-requirements) - + [Python 3.9](#python-39) -9. [Install the Chromedriver](#install-the-chromedriver) -10. [Install Google Chrome needed by Selenium](#install-google-chrome-needed-by-selenium) -11. [ECCS2 Script](#eccs2-script) - * [Install](#install) - * [Configure](#configure) - * [Execute](#execute) -12. [ECCS2 API Server (UWSGI)](#eccs2-api-server-uwsgi) - * [Install](#install-1) - * [Configure](#configure-1) - * [Utility](#utility) -13. [ECCS2 API JSON](#eccs2-api-json) -14. [Utility for web interface](#utility-for-web-interface) -15. [Utility for developers](#utility-for-developers) - * [ECCS2 API Development Server](#eccs2-api-development-server) -16. [Authors](#authors) + + [Install](#install) + * [Install the Chromedriver](#install-the-chromedriver) + * [Install Google Chrome needed by Selenium](#install-google-chrome-needed-by-selenium) + * [ECCS Script](#eccs-script) + + [Install](#install-1) + + [Configure](#configure) + + [Execute](#execute) +9. [ECCS API Server (UWSGI)](#eccs-api-server-uwsgi) + * [Install](#install-1) + * [Configure](#configure-1) + * [Utility](#utility) +10. [ECCS API JSON](#eccs-api-json) +11. [Utility for web interface](#utility-for-web-interface) +12. [Utility for developers](#utility-for-developers) + * [ECCS API Development Server](#eccs-api-development-server) +13. [Authors](#authors) # Introduction @@ -65,7 +65,7 @@ Disallow: / # On-line interface -The test eduGAIN Connectivity Check web pages is available at: https://technical-test.edugain.org/eccs2 +The test eduGAIN Connectivity Check web pages is available at: https://technical-test.edugain.org/eccs The tool uses following status for IdPs: @@ -95,18 +95,18 @@ The tool uses following status for IdPs: # Requirements Software * Apache Server + WSGI -* Python 3.9 (tested with v3.9.1) +* Python 3 (tested with v3.9.1, v3.9.7) * Selenim + Google Chrome Web Brower (tested with v91.0.4472.164) * Chromedriver (tested with v91.0.4472.101) * Git # HOWTO Install and Configure -## Download ECCS2 Repository +## Download ECCS Repository -* `cd $HOME ; git clone https://gitlab.geant.org/marco.malavolti/eccs2.git` +* `cd $HOME ; git clone https://gitlab.geant.org/edugain/eccs.git` -## Install Python 3.9.x +## Install Python 3 ### CentOS 7 requirements @@ -133,42 +133,42 @@ The tool uses following status for IdPs: 3. Install Git: * `sudo apt install git` -### Python 3.9 +### Install -1. Download the last version of Python 3.9.x from https://www.python.org/downloads/source/ into your home: - * `wget https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz -O $HOME/eccs2/Python-3.9.1.tgz` +1. Download the last version of Python 3 from https://www.python.org/downloads/source/ into your home: + * `wget https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz -O $HOME/eccs/Python-3.9.7.tgz` 2. Extract Python source package: - * `cd $HOME/eccs2/` - * `tar xzf Python-3.9.1.tgz` + * `cd $HOME/eccs/` + * `tar xzf Python-3.9.7.tgz` 3. Build Python from the source package: - * `cd $HOME/eccs2/Python-3.9.1` - * `./configure --prefix=$HOME/eccs2/python` + * `cd $HOME/eccs/Python-3.9.7` + * `./configure --prefix=$HOME/eccs/python` * `make` -4. Install Python 3.9.x under `$HOME/eccs2/python`: +4. Install Python 3 under `$HOME/eccs/python`: * `make install` - * `$HOME/eccs2/python/bin/python3.9 --version` + * `$HOME/eccs/python/bin/python3 --version` - This will install python under your $HOME directory. + This will install python3 under your $HOME/eccs directory. 5. Remove useless things: - * `rm -Rf $HOME/eccs2/Python-3.9.1 $HOME/eccs2/Python-3.9.1.tgz` + * `rm -Rf $HOME/eccs/Python-3.9.7 $HOME/eccs/Python-3.9.7.tgz` -# Install Google Chrome needed by Selenium +## Install Google Chrome needed by Selenium * Debian (64 bit): - * `cd $HOME/eccs2` + * `cd $HOME/eccs` * `sudo wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb` * `sudo apt install ./google-chrome-stable_current_amd64.deb` * CentOS (64 bit): - * `cd $HOME/eccs2` + * `cd $HOME/eccs` * `sudo wget https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm` * `sudo yum install ./google-chrome-stable_current_x86_64.rpm` -# Install the Chromedriver +## Install the Chromedriver 1. Find out which version of Chromium you are using: * Debian 9 (stretch): @@ -183,7 +183,7 @@ The tool uses following status for IdPs: 4. Use the version number retrieved from the previous step to construct the URL to download ChromeDriver. With version `72.0.3626.69`, the URL would be "https://chromedriver.storage.googleapis.com/index.html?path=73.0.3683.68/" 5. Download the Chromedriver and extract it with: - * `cd $HOME/eccs2` + * `cd $HOME/eccs` * `wget https://chromedriver.storage.googleapis.com/73.0.3683.75/chromedriver_linux64.zip` * `unzip chromedriver_linux64.zip` * `rm chromedriver_linux64.zip google-chrome-stable_current_amd64.deb` @@ -191,21 +191,21 @@ The tool uses following status for IdPs: **Note:** After the initial download, it is recommended that you occasionally go through the above process again to see if there are any bug fix releases. -# ECCS2 Script +## ECCS Script -## Install +### Install and Configure the Virtual Environment -* `cd $HOME/eccs2` -* `./python/bin/python3.9 -m pip install virtualenv` -* `$HOME/eccs2/python/bin/virtualenv --python=$HOME/eccs2/python/bin/python3.9 eccs2venv` -* `$HOME/eccs2/eccs2venv/bin/python -m pip install --upgrade pip` -* `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv) - * `python -m pip install -r requirements.txt` +* `cd $HOME/eccs` +* `./python/bin/python3 -m pip install virtualenv` +* `$HOME/eccs/python/bin/virtualenv --python=$HOME/eccs/python/bin/python3 eccs-venv` +* `$HOME/eccs/eccs-venv/bin/python3 -m pip install --upgrade pip` +* `source eccs-venv/bin/activate` (`deactivate` to exit Virtualenv) + * `python3 -m pip install -r requirements.txt` -## Configure +### Configure ECCS -1. Configure ECCS2 properties: - * `vim eccs2properties.py` (and change it upon your needs) +1. Configure ECCS properties: + * `vim eccs_properties.py` (and change it upon your needs) 2. Change `PATH` by adding the virtualenv Python `bin` dir: * CentOS: @@ -213,9 +213,9 @@ After the initial download, it is recommended that you occasionally go through t * Add the following lines at the tail: ```bash - # set PATH for ECCS2 - if [ -d "$HOME/eccs2" ] ; then - PATH="$HOME/eccs2/eccs2venv/bin:$PATH" + # set PATH for ECCS + if [ -d "$HOME/eccs" ] ; then + PATH="$HOME/eccs/eccs-venv/bin:$PATH" fi ``` @@ -226,38 +226,38 @@ After the initial download, it is recommended that you occasionally go through t * Add the following lines at the tail: ```bash - # set PATH for ECCS2 - if [ -d "$HOME/eccs2" ] ; then - PATH="$HOME/eccs2/eccs2venv/bin:$PATH" + # set PATH for ECCS + if [ -d "$HOME/eccs" ] ; then + PATH="$HOME/eccs/eccs-venv/bin:$PATH" fi ``` * `source $HOME/.bash_profile` -3. Configure ECCS2 cron job for the local user: +3. Configure ECCS cron job for the local user: * `crontab -e` ```bash SHELL=/bin/bash - 0 4 * * * /bin/bash $HOME/eccs2/cleanAndRunEccs2.sh > $HOME/eccs2/logs/eccs2cron.log 2>&1 + 0 4 * * * /bin/bash $HOME/eccs/cleanAndRunEccs.sh > $HOME/eccs/logs/eccs-cron.log 2>&1 ``` ## Execute - * `cd $HOME/eccs2` - * `./cleanAndRunEccs2.py` (to run a full and clean check) - * `./runEccs2.py` (to run a full check on the existing inputs) - * `./runEccs2.py --idp <IDP-ENTITYID>` (to run check on a single IdP) - * `./runEccs2.py --test` (to run a full check without effects) - * `./runEccs2.py --idp <IDP-ENTITYID> --test` (to run check on a single IdP without effects) - * `./runEccs2.py --idp <IDP-ENTITYID> --replace` (to run check on a single IdP and replace, or add, a result) + * `cd $HOME/eccs` + * `./cleanAndRunEccs.py` (to run a full and clean check) + * `./runEccs.py` (to run a full check on the existing inputs) + * `./runEccs.py --idp <IDP-ENTITYID>` (to run check on a single IdP) + * `./runEccs.py --test` (to run a full check without effects) + * `./runEccs.py --idp <IDP-ENTITYID> --test` (to run check on a single IdP without effects) + * `./runEccs.py --idp <IDP-ENTITYID> --replace` (to run check on a single IdP and replace, or add, a result) - If something prevent the good execution of the ECCS2's check, the `logs/failed-cmd.sh` file will be not empty at the end of the execution. + If something prevent the good execution of the ECCS's check, the `logs/failed-cmd.sh` file will be not empty at the end of the execution. - The "--test" parameter will not change the result of ECCS2, but will write the output on the `logs/stdout_idp_YYYY-MM-DD.log`,`logs/stderr_idp_YYYY-MM-DD.log` and `logs/failed-cmd-idp.sh` files if the argument "--test" will be used. + The "--test" parameter will not change the result of ECCS, but will write the output on the `logs/stdout_idp_YYYY-MM-DD.log`,`logs/stderr_idp_YYYY-MM-DD.log` and `logs/failed-cmd-idp.sh` files if the argument "--test" will be used. -# ECCS2 API Server (uWSGI) +# ECCS API Server (uWSGI) ## Install @@ -266,32 +266,32 @@ After the initial download, it is recommended that you occasionally go through t * `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python3-dev unzip` * CentOS: * `sudo yum install mod_proxy_uwsgi unzip` - * Configure SElinux to enable ECCS2: - * `sudo semanage fcontext -a -t httpd_sys_content_t "$HOME/eccs2(/.*)?"` - * `sudo restorecon -R -e $HOME/eccs2/` + * Configure SElinux to enable ECCS: + * `sudo semanage fcontext -a -t httpd_sys_content_t "$HOME/eccs(/.*)?"` + * `sudo restorecon -R -e $HOME/eccs/` * `sudo setsebool -P httpd_can_network_connect 1` ## Configure -1. Add the systemd service to enable ECCS2 API: - * `cd $HOME/eccs2` - * `cp eccs2.ini.template eccs2.ini` - * `cp eccs2.service.template eccs2.service` - * `vim eccs2.ini` (and change "`uid`", "`gid`" and "`base`" values opportunely) - * `vim eccs2.service` (and change "`User`","`Group`","`WorkingDirectory`","`RuntimeDirectory`","`ExecStart`" values opportunely) - * `sudo cp $HOME/eccs2/eccs2.service /etc/systemd/system/eccs2.service` +1. Add the systemd service to enable ECCS API: + * `cd $HOME/eccs` + * `cp eccs.ini.template eccs.ini` + * `cp eccs.service.template eccs.service` + * `vim eccs.ini` (and change "`uid`", "`gid`" and "`base`" values opportunely) + * `vim eccs.service` (and change "`User`","`Group`","`WorkingDirectory`","`RuntimeDirectory`","`ExecStart`" values opportunely) + * `sudo cp $HOME/eccs/eccs.service /etc/systemd/system/eccs.service` * `sudo systemctl daemon-reload` - * `sudo systemctl enable eccs2.service` - * `sudo systemctl start eccs2.service` + * `sudo systemctl enable eccs.service` + * `sudo systemctl start eccs.service` -2. Configure Apache for ECCS2 web side: +2. Configure Apache for ECCS web side: * Debian: - * `sudo cp $HOME/eccs2/eccs2-debian.conf /etc/apache2/conf-available/eccs2.conf` - * `sudo a2enconf eccs2.conf` + * `sudo cp $HOME/eccs/eccs-debian.conf /etc/apache2/conf-available/eccs.conf` + * `sudo a2enconf eccs.conf` * `sudo chgrp www-data $HOME ; sudo chmod g+rx $HOME` (Apache needs permission to access the $HOME dir) * `sudo systemctl restart apache2.service` * CentOS: - * `sudo cp $HOME/eccs2/eccs2-centos.conf /etc/httpd/conf.d/eccs2.conf` + * `sudo cp $HOME/eccs/eccs-centos.conf /etc/httpd/conf.d/eccs.conf` * `sudo chgrp apache $HOME ; sudo apache g+rx $HOME` (Apache needs permission to access the $HOME dir) * `sudo systemctl restart httpd.service` @@ -301,22 +301,22 @@ After the initial download, it is recommended that you occasionally go through t ```bash SHELL=/bin/bash - 0 3 * * * /usr/bin/touch $HOME/eccs2/eccs2.ini + 0 3 * * * /usr/bin/touch $HOME/eccs/eccs.ini ``` ## Utility To perform a restart after an API change use the following command: -* `touch $HOME/eccs2/eccs2.ini` +* `touch $HOME/eccs/eccs.ini` -# ECCS2 API JSON +# ECCS API JSON * `/api/eccsresults` (Return the results of the last check ready for ECCS web interface) * `/api/eccsresults?<parameter1>=<value1>&<parameter2>=<value2>`: * `date=2020-02-20` (select date) * `idp=https://idp.example.org/idp/shibboleth` (select a specific idp) - * `status=` (select specific ECCS2 status) + * `status=` (select specific ECCS status) * 'OK' * 'ERROR' * 'DISABLED' @@ -339,18 +339,18 @@ The available dates are provided by the first and the last file created into the ## Clean old results -To clean the ECCS2 results from files older than last 7 days use (modify it on your needs): +To clean the ECCS results from files older than last 7 days use (modify it on your needs): * `crontab -e` ```bash SHELL=/bin/bash - 0 10 * * * /bin/bash $HOME/eccs2/clean7daysOldFiles.sh > $HOME/eccs2/logs/clean7daysOldFiles.log 2>&1 + 0 10 * * * /bin/bash $HOME/eccs/clean7daysOldFiles.sh > $HOME/eccs/logs/clean7daysOldFiles.log 2>&1 ``` ## User interface -The eduGAIN Connectivity Check 2 test web page is available at https://technical-test.edugain.org/eccs2 +The eduGAIN Connectivity Check Service web page is available at https://technical-test.edugain.org/eccs ## User interface parameters @@ -364,13 +364,13 @@ The eduGAIN Connectivity Check 2 test web page is available at https://technical **Example:** -`https://technical-test.edugain.org/eccs2?reg_auth=http://www.idem.garr.it/&check_result=SSL-Error` +`https://technical-test.edugain.org/eccs?reg_auth=http://www.idem.garr.it/&check_result=SSL-Error` # Utility for developers -## ECCS2 API Development Server +## ECCS API Development Server -* `cd $HOME/eccs2 ; ./api.py` +* `cd $HOME/eccs ; ./api.py` # Authors diff --git a/api.py b/api.py index 66ccb14..cea39c6 100755 --- a/api.py +++ b/api.py @@ -4,7 +4,7 @@ import json import logging import re -from eccs2properties import DAY, ECCS2LOGSDIR, ECCS2OUTPUTDIR, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE +from eccs_properties import DAY, ECCS_LOGSDIR, ECCS_OUTPUTDIR, ECCS_LISTFEDSURL, ECCS_LISTFEDSFILE, ECCS_RESULTSLOG from flask import Flask, request, jsonify from flask_restful import Resource, Api from utils import get_logger, get_list_feds, get_reg_auth_dict @@ -15,7 +15,7 @@ api = Api(app) ### Functions -# Build Email Addresses Link for ECCS2 Web Gui +# Build Email Addresses Link for ECCS Web Gui def buildEmailAddress(listContacts): listCtcs = listContacts.split(",") hrefList = [] @@ -25,7 +25,7 @@ def buildEmailAddress(listContacts): return hrefList -# Research the value of the research_item into ECCS2 output files +# Research the value of the research_item into ECCS output files def existsInFile(file_path, value, research_item, eccsDataTable, date): try: with open(file_path,"r",encoding="utf-8") as fo: @@ -34,7 +34,7 @@ def existsInFile(file_path, value, research_item, eccsDataTable, date): if (eccsDataTable): return '' else: - return jsonify(error=f'FileNotFound: ECCS2 script has not been executed on {date} yet') + return jsonify(error=f'FileNotFound: ECCS script has not been executed on {date} yet') for line in lines: aux = json.loads(line) @@ -70,7 +70,7 @@ class Test(Resource): class EccsResults(Resource): def get(self): - file_path = f"{ECCS2OUTPUTDIR}/eccs2_{DAY}.log" + file_path = f"{ECCS_OUTPUTDIR}/{ECCS_RESULTSLOG}" date = DAY status = None idp = None @@ -83,7 +83,7 @@ class EccsResults(Resource): eccsDataTable = True if 'date' in request.args: date = request.args['date'] - file_path = f"{ECCS2OUTPUTDIR}/eccs2_{date}.log" + file_path = f"{ECCS_OUTPUTDIR}/eccs_{date}.log" if 'status' in request.args: status = request.args['status'].upper() if (status not in ['OK','DISABLED','ERROR']): @@ -114,7 +114,7 @@ class EccsResults(Resource): if (eccsDataTable): return '' else: - return jsonify(error=f'FileNotFound: ECCS2 script has not been executed on {date}') + return jsonify(error=f'FileNotFound: ECCS script has not been executed on {date}') for line in lines: # Strip the line feed and carriage return characters @@ -125,7 +125,7 @@ class EccsResults(Resource): aux['date'] = date - # If the results are for ECCS2 DataTable, otherwise... remove only "mailto:" prefix + # If the results are for ECCS DataTable, otherwise... remove only "mailto:" prefix if (eccsDataTable): aux['contacts']['technical'] = buildEmailAddress(aux['contacts']['technical']) aux['contacts']['support'] = buildEmailAddress(aux['contacts']['support']) @@ -195,17 +195,17 @@ class EccsResults(Resource): # /api/fedstats class FedStats(Resource): def get(self): - list_feds = get_list_feds(ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE) + list_feds = get_list_feds(ECCS_LISTFEDSURL, ECCS_LISTFEDSFILE) regAuthDict = get_reg_auth_dict(list_feds) - file_path = f"{ECCS2OUTPUTDIR}/eccs2_{DAY}.log" + file_path = f"{ECCS_OUTPUTDIR}/{ECCS_RESULTSLOG}" date = DAY reg_auth = None eccsDataTable = False if ('date' in request.args): date = request.args['date'] - file_path = f"{ECCS2OUTPUTDIR}/eccs2_{date}.log" + file_path = f"{ECCS_OUTPUTDIR}/eccs_{date}.log" if ('reg_auth' in request.args): reg_auth = request.args['reg_auth'] if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable, date)): @@ -221,7 +221,7 @@ class FedStats(Resource): if (eccsDataTable): return '' else: - return jsonify(error=f'FileNotFound: ECCS2 script has not been executed on {date} yet') + return jsonify(error=f'FileNotFound: ECCS script has not been executed on {date} yet') if (reg_auth): resultDict = {'date': date, 'registrationAuthority': reg_auth, 'OK': 0, 'ERROR': 0, 'DISABLED': 0} @@ -269,7 +269,7 @@ class FedStats(Resource): # /api/ class Help(Resource): def get(self): - return { 'ECCS2 JSON Interface': 'https://wiki.geant.org/display/eduGAIN/eduGAIN+Connectivity+Check+2#eduGAINConnectivityCheck2-JSONinterface' } + return { 'ECCS JSON Interface': 'https://wiki.geant.org/display/eduGAIN/eduGAIN+Connectivity+Check+2#eduGAINConnectivityCheck2-JSONinterface' } # Routes @@ -283,5 +283,5 @@ if __name__ == '__main__': # Useful only for API development Server #app.config['JSON_AS_ASCII'] = True #app.logger.removeHandler(default_handler) - #app.logger = get_logger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO") + #app.logger = get_logger("eccs_api.log", ECCS_LOGSDIR, "w", "INFO") app.run(port='5002') diff --git a/clean7daysOldFiles.sh b/clean7daysOldFiles.sh index 86ee441..1744268 100755 --- a/clean7daysOldFiles.sh +++ b/clean7daysOldFiles.sh @@ -1,6 +1,6 @@ #!/bin/bash -BASEDIR=$HOME/eccs2 +BASEDIR=$HOME/eccs # Remove ECCS2 result older than 7 days find $BASEDIR/output/* -mtime +6 -type f -delete diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs.sh similarity index 69% rename from cleanAndRunEccs2.sh rename to cleanAndRunEccs.sh index 9242c5d..cfb7fa1 100755 --- a/cleanAndRunEccs2.sh +++ b/cleanAndRunEccs.sh @@ -7,10 +7,10 @@ BASEDIR=$HOME source $HOME/.bash_profile # Remove old IdP and Fed List -rm -f $BASEDIR/eccs2/input/*.json +rm -f $BASEDIR/eccs/input/*.json # Run ECCS2 -$BASEDIR/eccs2/runEccs2.py +$BASEDIR/eccs/runEccs.py # Run again ECCS2 for those IdPs who failed check -$BASEDIR/eccs2/retryFailedChecks.py +$BASEDIR/eccs/retryFailedChecks.py diff --git a/eccs2-centos.conf b/eccs-centos.conf similarity index 50% rename from eccs2-centos.conf rename to eccs-centos.conf index 4277791..c83356e 100644 --- a/eccs2-centos.conf +++ b/eccs-centos.conf @@ -1,16 +1,16 @@ <IfModule mod_alias.c> - Alias /eccs2 /home/<USER>/eccs2/web - Alias /eccs2html /home/<USER>/eccs2/html + Alias /eccs /home/<USER>/eccs/web + Alias /eccs/html /home/<USER>/eccs/html - <Directory /home/<USER>/eccs2/web> + <Directory /home/<USER>/eccs/web> DirectoryIndex index.php Require all granted </Directory> - <Directory /home/<USER>/eccs2/html> + <Directory /home/<USER>/eccs/html> Require all granted </Directory> </IfModule> LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so -ProxyPass /eccs2/api uwsgi://127.0.0.1:8000/ +ProxyPass /eccs/api uwsgi://127.0.0.1:8000/ diff --git a/eccs-debian.conf b/eccs-debian.conf new file mode 100644 index 0000000..63b8aa3 --- /dev/null +++ b/eccs-debian.conf @@ -0,0 +1,17 @@ +<IfModule mod_alias.c> + Alias /eccs /home/<USER>/eccs/web + Alias /eccs/html /home/<USER>/eccs/html + + <Directory /home/<USER>/eccs/web> + DirectoryIndex index.php + Require all granted + </Directory> + + <Directory /home/<USER>/eccs/html> + Require all granted + </Directory> +</IfModule> + +<IfModule mod_proxy_uwsgi.c> + ProxyPass /eccs/api uwsgi://127.0.0.1:8000/ +</IfModule> diff --git a/eccs2wsgi.py b/eccs-wsgi.py similarity index 77% rename from eccs2wsgi.py rename to eccs-wsgi.py index c6dc305..796d847 100644 --- a/eccs2wsgi.py +++ b/eccs-wsgi.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3 from api import app as application diff --git a/eccs2.ini.template b/eccs.ini.template similarity index 73% rename from eccs2.ini.template rename to eccs.ini.template index 3ab090f..6b2d623 100644 --- a/eccs2.ini.template +++ b/eccs.ini.template @@ -1,5 +1,5 @@ [uwsgi] -project = eccs2 +project = eccs base = /home/<USER> chdir = %(base)/%(project) @@ -16,15 +16,15 @@ vacuum = true die-on-term = true -# set max connections to 1024 in uWSGI +# set max connections to 128 in uWSGI listen = 128 -virtualenv = eccs2venv -wsgi-file = eccs2wsgi.py +virtualenv = eccs-venv +wsgi-file = eccs-wsgi.py -logto = logs/%(project)uwsgi.log +logto = logs/%(project)-uwsgi.log log-maxsize = 100000000 -log-backupname = logs/%(project)uwsgi.old.log +log-backupname = logs/%(project)-uwsgi.old.log # Each 1000 request restart workers max-requests = 1000 diff --git a/eccs2.py b/eccs.py similarity index 90% rename from eccs2.py rename to eccs.py index 83e7aba..cc84023 100755 --- a/eccs2.py +++ b/eccs.py @@ -4,13 +4,13 @@ import argparse import json import sys import utils -import eccs2properties as e2p +import eccs_properties as e_p from pathlib import Path """ The check works with the wayfless url of two SP and successed if the IdP Login Page appears and contains the fields "username" and "password" for each of them. -It is possible to disable the check by eccs2properties IDP_DISABLE_DICT or by "robots.txt" put on the SAMLRequest endpoint root web dir. +It is possible to disable the check by eccs_properties IDP_DISABLE_DICT or by "robots.txt" put on the SAMLRequest endpoint root web dir. """ # Extract IdP DisplayName by fixing input string @@ -35,7 +35,7 @@ def store_eccs_result(idp,sp,check_results,idp_status,test): str_support_contacts = ','.join(list_support_contacts) if (test): - sys.stdout.write("\nECCS2:") + sys.stdout.write("\nECCS:") sys.stdout.write('{"displayName":"%s","entityID":"%s","registrationAuthority":"%s","contacts":{"technical":"%s","support":"%s"},"status":"%s","sp1":{"wayflessUrl":"%s","checkTime":"%s","checkResult":"%s"},"sp2":{"wayflessUrl":"%s","checkTime":"%s","checkResult":"%s"}}\n' % ( get_display_name(idp['displayname']), # IdP-DisplayName idp['entityID'], # IdP-entityID @@ -52,7 +52,7 @@ def store_eccs_result(idp,sp,check_results,idp_status,test): else: # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;IdP-ECCS-Status;SP-wayfless-url-1;SP-check-time-1;SP-result-1;SP-wayfless-url-2;SP-check-time-2;SP-result-2 - with open(f"{e2p.ECCS2OUTPUTDIR}/{e2p.ECCS2RESULTSLOG}", 'a') as f: + with open(f"{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}", 'a') as f: try: f.write('{"displayName":"%s","entityID":"%s","registrationAuthority":"%s","contacts":{"technical":"%s","support":"%s"},"status":"%s","sp1":{"wayflessUrl":"%s","checkTime":"%s","checkResult":"%s"},"sp2":{"wayflessUrl":"%s","checkTime":"%s","checkResult":"%s"}}\n' % ( get_display_name(idp['displayname']), # IdP-DisplayName @@ -70,21 +70,21 @@ def store_eccs_result(idp,sp,check_results,idp_status,test): ) ) except IOError: - sys.stderr.write(f"Failed writing result on output file for {idp['entityID']} with {utils.get_label(sp)}.\n\nRun {e2p.ECCS2DIR}/runEccs2.py --idp {idp['entityID']} --replace\n") + sys.stderr.write(f"Failed writing result on output file for {idp['entityID']} with {utils.get_label(sp)}.\n\nRun {e_p.ECCS_DIR}/runEccs.py --idp {idp['entityID']} --replace\n") sys.exit(1) # Check an IdP with 2 SPs. def check(idp,test): check_results = [] - for sp in e2p.ECCS2SPS: + for sp in e_p.ECCS_SPS: result = utils.check_idp_response_selenium(sp,idp,test) if (result): check_results.append(result) else: - sys.stderr.write(f"\nCheck failed for {idp['entityID']} with {utils.get_label(sp)}.\n\nRun {e2p.ECCS2DIR}/runEccs2.py --idp {idp['entityID']} --replace\n") + sys.stderr.write(f"\nCheck failed for {idp['entityID']} with {utils.get_label(sp)}.\n\nRun {e_p.ECCS_DIR}/runEccs.py --idp {idp['entityID']} --replace\n") sys.exit(1) - if (len(check_results) == len(e2p.ECCS2SPS)): + if (len(check_results) == len(e_p.ECCS_SPS)): check_result_sp1 = check_results[0][3] check_result_sp2 = check_results[1][3] check_result_weberr1 = check_results[0][4] @@ -112,9 +112,9 @@ if __name__=="__main__": idp = json.loads(args.idpJson[0]) - Path(f"{e2p.ECCS2HTMLDIR}/{e2p.DAY}").mkdir(parents=True, exist_ok=True) # Create dir needed to page_source content + Path(f"{e_p.ECCS_HTMLDIR}/{e_p.DAY}").mkdir(parents=True, exist_ok=True) # Create dir needed to page_source content if (args.replace and not args.test): - utils.delete_line_with_word(f"{e2p.ECCS2OUTPUTDIR}/{e2p.ECCS2RESULTSLOG}",idp['entityID']) + utils.delete_line_with_word(f"{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}",idp['entityID']) check(idp,args.test) diff --git a/eccs.service.template b/eccs.service.template new file mode 100644 index 0000000..54dff5b --- /dev/null +++ b/eccs.service.template @@ -0,0 +1,18 @@ +[Install] +WantedBy=multi-user.target + +[Unit] +Description=uWSGI server for ECCS +After=network.target + +[Service] +User=<USER> +Group=<USER> +WorkingDirectory=/home/<USER>/eccs +RuntimeDirectory=/home/<USER>/eccs +Restart=always +KillSignal=SIGQUIT +Type=notify +StandardError=inherit +NotifyAccess=all +ExecStart=/home/<USER>/eccs/eccs-venv/bin/uwsgi --ini /home/<USER>/eccs/eccs.ini diff --git a/eccs2-debian.conf b/eccs2-debian.conf deleted file mode 100644 index 539e9a0..0000000 --- a/eccs2-debian.conf +++ /dev/null @@ -1,17 +0,0 @@ -<IfModule mod_alias.c> - Alias /eccs2 /home/<USER>/eccs2/web - Alias /eccs2html /home/<USER>/eccs2/html - - <Directory /home/<USER>/eccs2/web> - DirectoryIndex index.php - Require all granted - </Directory> - - <Directory /home/<USER>/eccs2/html> - Require all granted - </Directory> -</IfModule> - -<IfModule mod_proxy_uwsgi.c> - ProxyPass /eccs2/api uwsgi://127.0.0.1:8000/ -</IfModule> diff --git a/eccs2.service.template b/eccs2.service.template deleted file mode 100644 index bf572c3..0000000 --- a/eccs2.service.template +++ /dev/null @@ -1,18 +0,0 @@ -[Install] -WantedBy=multi-user.target - -[Unit] -Description=uWSGI server for ECCS2 -After=network.target - -[Service] -User=<USER> -Group=<USER> -WorkingDirectory=/home/<USER>/eccs2 -RuntimeDirectory=/home/<USER>/eccs2 -Restart=always -KillSignal=SIGQUIT -Type=notify -StandardError=inherit -NotifyAccess=all -ExecStart=/home/<USER>/eccs2/eccs2venv/bin/uwsgi --ini /home/<USER>/eccs2/eccs2.ini diff --git a/eccs2properties.py b/eccs_properties.py similarity index 69% rename from eccs2properties.py rename to eccs_properties.py index d12653f..26b9cb4 100644 --- a/eccs2properties.py +++ b/eccs_properties.py @@ -3,49 +3,49 @@ from datetime import date DAY = date.today().isoformat() -ECCS2DIR = f"{os.environ['HOME']}/eccs2" -PATHCHROMEDRIVER = f"{ECCS2DIR}/chromedriver" -ECCS2PYTHON = f"{ECCS2DIR}/python/bin/python3" +ECCS_DIR = f"{os.environ['HOME']}/eccs" +PATHCHROMEDRIVER = f"{ECCS_DIR}/chromedriver" +ECCS_PYTHON = f"{ECCS_DIR}/python/bin/python3" # Input -ECCS2INPUTDIR = f"{ECCS2DIR}/input" -ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json' -ECCS2LISTIDPSFILE = f"{ECCS2INPUTDIR}/list_eccs_idps.json" -ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' -ECCS2LISTFEDSFILE = f"{ECCS2INPUTDIR}/list_fed.json" +ECCS_INPUTDIR = f"{ECCS_DIR}/input" +ECCS_LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json' +ECCS_LISTIDPSFILE = f"{ECCS_INPUTDIR}/list_eccs_idps.json" +ECCS_LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' +ECCS_LISTFEDSFILE = f"{ECCS_INPUTDIR}/list_fed.json" # Output -ECCS2OUTPUTDIR = f"{ECCS2DIR}/output" -ECCS2RESULTSLOG = f"eccs2_{DAY}.log" -ECCS2HTMLDIR = f"{ECCS2DIR}/html" +ECCS_OUTPUTDIR = f"{ECCS_DIR}/output" +ECCS_RESULTSLOG = f"eccs_{DAY}.log" +ECCS_HTMLDIR = f"{ECCS_DIR}/html" # Selenium -ECCS2SELENIUMDEBUG = False -ECCS2SELENIUMLOGDIR = f"{ECCS2DIR}/selenium-logs" -ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds (remind to change timeout seconds also on web/eccs2.js) -ECCS2SELENIUMSCRIPTTIMEOUT = 30 #seconds -ECCS2REQUESTSTIMEOUT = 15 #seconds +ECCS_SELENIUMDEBUG = False +ECCS_SELENIUMLOGDIR = f"{ECCS_DIR}/selenium-logs" +ECCS_SELENIUMPAGELOADTIMEOUT = 30 #seconds (remind to change timeout seconds also on web/eccs.js) +ECCS_SELENIUMSCRIPTTIMEOUT = 30 #seconds +ECCS_REQUESTSTIMEOUT = 15 #seconds # Logs -ECCS2LOGSDIR = f"{ECCS2DIR}/logs" -ECCS2STDOUT = f"{ECCS2LOGSDIR}/stdout_{DAY}.log" -ECCS2STDERR = f"{ECCS2LOGSDIR}/stderr_{DAY}.log" -ECCS2FAILEDCMD = f"{ECCS2LOGSDIR}/failed-cmd.sh" -ECCS2STDOUTIDP = f"{ECCS2LOGSDIR}/stdout_idp_{DAY}.log" -ECCS2STDERRIDP = f"{ECCS2LOGSDIR}/stderr_idp_{DAY}.log" -ECCS2FAILEDCMDIDP = f"{ECCS2LOGSDIR}/failed-cmd-idp.sh" +ECCS_LOGSDIR = f"{ECCS_DIR}/logs" +ECCS_STDOUT = f"{ECCS_LOGSDIR}/stdout_{DAY}.log" +ECCS_STDERR = f"{ECCS_LOGSDIR}/stderr_{DAY}.log" +ECCS_FAILEDCMD = f"{ECCS_LOGSDIR}/failed-cmd.sh" +ECCS_STDOUTIDP = f"{ECCS_LOGSDIR}/stdout_idp_{DAY}.log" +ECCS_STDERRIDP = f"{ECCS_LOGSDIR}/stderr_idp_{DAY}.log" +ECCS_FAILEDCMDIDP = f"{ECCS_LOGSDIR}/failed-cmd-idp.sh" # Number of processes to run in parallel -ECCS2NUMPROCESSES = 35 +ECCS_NUMPROCESSES = 35 # The 2 SPs that will be used to test each IdP -ECCS2SPS = [ +ECCS_SPS = [ "https://sp-demo.idem.garr.it/Shibboleth.sso/Login?entityID=", "https://attribute-viewer.aai.switch.ch/interfederation-test/Shibboleth.sso/Login?entityID=" ] # ROBOTS.TXT -ROBOTS_USER_AGENT = "ECCS/2.0 (+https://technical-test.edugain.org/eccs2)" +ROBOTS_USER_AGENT = "ECCS/2.0 (+https://technical.edugain.org/eccs)" # PATTERNS METADATAPATTERN = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements|Unsupported.Request|Not.Authorized|METADATANOTFOUND|Unknown.login.requester|is.unspecified.or.unsupported|Unknown.service.provider|Richiesta.non.supportata|Metadati.non.trovati|untrusted.provider|Unregistered.Service|Unsupported.request|UNHANDLEDEXCEPTION|Metadata.*.expired|Could.not.find.any.*.metadata.*.for" @@ -62,10 +62,10 @@ FEDS_DISABLED_DICT = { # { 'entityid_idp':'reason' } IDPS_DISABLED_DICT = { - 'https://idp.eie.gr/idp/shibboleth':'Disabled on 2019-04-24 because ECCS2 cannot check non-standard login page', + 'https://idp.eie.gr/idp/shibboleth':'Disabled on 2019-04-24 because ECCS cannot check non-standard login page', 'https://edugain-proxy.igtf.net/simplesaml/saml2/idp/metadata.php':'Disabled on 2017-03-17 on request of federation operator', - 'https://gn-vho.grnet.gr/idp/shibboleth':'Disabled on 2019-04-24 because basic authentication is not supported by ECCS2 check', - 'https://wtc.tu-chemnitz.de/shibboleth':'Disabled on 2019-02-26 because ECCS2 cannot check non-standard login page', + 'https://gn-vho.grnet.gr/idp/shibboleth':'Disabled on 2019-04-24 because basic authentication is not supported by ECCS check', + 'https://wtc.tu-chemnitz.de/shibboleth':'Disabled on 2019-02-26 because ECCS cannot check non-standard login page', 'https://idp.fraunhofer.de/idp/shibboleth':'Disabled on 2017-11-24 on request of federation operator', 'https://idp.dfn-cert.de/idp/shibboleth':'Disabled on 2018-04-05 on request of federation operator', 'https://idp.cambria.ac.uk/openathens':'Disabled on 2017-10-27 on request of federation operator', @@ -78,7 +78,7 @@ IDPS_DISABLED_DICT = { 'https://idp.uel.ac.uk/shibboleth':'Disabled on 2017-10-27 on request of federation operator', 'https://idp.ucreative.ac.uk/shibboleth':'Disabled on 2017-10-27 on request of federation operator', 'https://idp.llandrillo.ac.uk/shibboleth':'Disabled on 2017-10-27 on request of federation operator', - 'https://sso.vu.lt/SSO/saml2/idp/metadata.php':'Disabled on 2018-11-02 because ECCS2 cannot check non-standard login page', - 'https://ssl.education.lu/saml/saml2/idp/metadata.php':'Disabled on 2018-11-06 ECCS2 cannot check non-standard login page', - 'https://sso.oktaedu.com/idp/shibboleth':'Disabled on 2021-08-12 because ECCS2 cannot check non-standard login page', + 'https://sso.vu.lt/SSO/saml2/idp/metadata.php':'Disabled on 2018-11-02 because ECCS cannot check non-standard login page', + 'https://ssl.education.lu/saml/saml2/idp/metadata.php':'Disabled on 2018-11-06 ECCS cannot check non-standard login page', + 'https://sso.oktaedu.com/idp/shibboleth':'Disabled on 2021-08-12 because ECCS cannot check non-standard login page', } diff --git a/retryFailedChecks.py b/retryFailedChecks.py index 482d0bc..424d525 100755 --- a/retryFailedChecks.py +++ b/retryFailedChecks.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import os -import eccs2properties as e2p +import eccs_properties as e_p import utils def get_idp_entityID(line): import json - line = line.lstrip(f"{e2p.ECCS2DIR}/eccs2.py '") + line = line.lstrip(f"{e_p.ECCS_DIR}/eccs.py '") line = line.rstrip("\'\n") json_line = json.loads(line) @@ -17,25 +17,25 @@ def get_idp_entityID(line): # MAIN if __name__=="__main__": - if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0): - print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.") + if (os.stat(e_p.ECCS_FAILEDCMD).st_size == 0): + print(f"{e_p.DAY} - ECCS OK: All eduGAIN IdPs have been checked successfully.") else: - with open(e2p.ECCS2FAILEDCMD) as f: + with open(e_p.ECCS_FAILEDCMD) as f: - # For each one, run ECCS2 check and remove its line + # For each one, run ECCS check and remove its line # from the "failed-cmd.sh". for line in f: idp = get_idp_entityID(line) os.system(f'{line.rstrip()}') - with open(f'{e2p.ECCS2OUTPUTDIR}/eccs2_{e2p.DAY}.log') as o: + with open(f'{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}') as o: for line in o: if (idp in line): - utils.delete_line_with_word(e2p.ECCS2FAILEDCMD,idp) - print(f"ECCS2 check retried successfully for {idp}") + utils.delete_line_with_word(e_p.ECCS_FAILEDCMD,idp) + print(f"ECCS check retried successfully for {idp}") - if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0): - print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.") + if (os.stat(e_p.ECCS_FAILEDCMD).st_size == 0): + print(f"{e_p.DAY} - ECCS OK: All eduGAIN IdPs have been checked successfully.") else: - print(f"{e2p.DAY} - Something went wrong. See the log files and failed-cmd.sh.") + print(f"{e_p.DAY} - Something went wrong. See the log files and failed-cmd.sh.") diff --git a/runEccs2.py b/runEccs.py similarity index 79% rename from runEccs2.py rename to runEccs.py index a0a7282..0b60518 100755 --- a/runEccs2.py +++ b/runEccs.py @@ -6,7 +6,7 @@ import datetime import json import time -import eccs2properties as e2p +import eccs_properties as e_p import utils from subprocess import PIPE @@ -52,7 +52,7 @@ async def main(cmd_list,stdout_file,stderr_file,cmd_file): # Create worker tasks to process the queue concurrently. tasks = [] - for i in range(e2p.ECCS2NUMPROCESSES): + for i in range(e_p.ECCS_NUMPROCESSES): task = asyncio.create_task(run(f"cmd-{i}", queue, stdout_file, stderr_file, cmd_file)) tasks.append(task) @@ -81,25 +81,25 @@ if __name__=="__main__": start = time.time() # Setup list_feds - url = e2p.ECCS2LISTFEDSURL - dest_file = e2p.ECCS2LISTFEDSFILE + url = e_p.ECCS_LISTFEDSURL + dest_file = e_p.ECCS_LISTFEDSFILE list_feds = utils.get_list_feds(url, dest_file) # Setup list_eccs_idps - url = e2p.ECCS2LISTIDPSURL - dest_file = e2p.ECCS2LISTIDPSFILE + url = e_p.ECCS_LISTIDPSURL + dest_file = e_p.ECCS_LISTIDPSFILE list_eccs_idps = utils.get_list_eccs_idps(url, dest_file) if (args.idp_entityid): - stdout_file = open(e2p.ECCS2STDOUTIDP,"w+") - stderr_file = open(e2p.ECCS2STDERRIDP,"w+") - cmd_file = open(e2p.ECCS2FAILEDCMDIDP,"w+") + stdout_file = open(e_p.ECCS_STDOUTIDP,"w+") + stderr_file = open(e_p.ECCS_STDERRIDP,"w+") + cmd_file = open(e_p.ECCS_FAILEDCMDIDP,"w+") idpJsonList = utils.get_idp_list(list_eccs_idps,idp_entityid=args.idp_entityid[0]) if (args.test): - cmd = f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idpJsonList[0])}' --test" + cmd = f"{e_p.ECCS_DIR}/eccs.py '{json.dumps(idpJsonList[0])}' --test" elif (args.replace): - cmd = f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idpJsonList[0])}' --replace" + cmd = f"{e_p.ECCS_DIR}/eccs.py '{json.dumps(idpJsonList[0])}' --replace" # List of only one command proc_list = [cmd] @@ -113,21 +113,21 @@ if __name__=="__main__": cmd_file.close() else: - stdout_file = open(e2p.ECCS2STDOUT,"w+") - stderr_file = open(e2p.ECCS2STDERR,"w+") - cmd_file = open(e2p.ECCS2FAILEDCMD,"w+") + stdout_file = open(e_p.ECCS_STDOUT,"w+") + stderr_file = open(e_p.ECCS_STDERR,"w+") + cmd_file = open(e_p.ECCS_FAILEDCMD,"w+") - # Prepare input file for ECCS2 + # Prepare input file for ECCS idpJsonList = utils.get_idp_list(list_eccs_idps) num_idps = len(idpJsonList) # Construct the list of commands to exec if (args.test): - cmd_list = [[f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idp)}' --test"] for idp in idpJsonList] + cmd_list = [[f"{e_p.ECCS_DIR}/eccs.py '{json.dumps(idp)}' --test"] for idp in idpJsonList] elif (args.replace): - cmd_list = [[f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idp)}' --replace"] for idp in idpJsonList] + cmd_list = [[f"{e_p.ECCS_DIR}/eccs.py '{json.dumps(idp)}' --replace"] for idp in idpJsonList] else: - cmd_list = [[f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idp)}'"] for idp in idpJsonList] + cmd_list = [[f"{e_p.ECCS_DIR}/eccs.py '{json.dumps(idp)}'"] for idp in idpJsonList] # String Convertion needed for Asyncio proc_list = [] diff --git a/utils.py b/utils.py index 00e51ce..cc89665 100644 --- a/utils.py +++ b/utils.py @@ -10,7 +10,7 @@ import sys import shutil import time -import eccs2properties as e2p +import eccs_properties as e_p from selenium import webdriver from selenium.common.exceptions import WebDriverException,TimeoutException,NoSuchElementException @@ -94,7 +94,7 @@ def get_list_eccs_idps(url, dest_file): return json.loads(f.read().replace("'","'")) -# Use logger to produce files consumed by ECCS-2 API +# Use logger to produce files consumed by ECCS API def get_logger(path, filename, mode="a", log_level="DEBUG"): logger = logging.getLogger(__name__) @@ -146,7 +146,7 @@ def store_page_source(page_source,idp,sp,test): return True else: # Put the page_source into an appropriate HTML file - with open(f"{e2p.ECCS2HTMLDIR}/{e2p.DAY}/{sha1(idp['entityID'])}---{get_label(sp)}.html","w") as html: + with open(f"{e_p.ECCS_HTMLDIR}/{e_p.DAY}/{sha1(idp['entityID'])}---{get_label(sp)}.html","w") as html: try: html.write(page_source) return True @@ -178,16 +178,16 @@ def get_driver_selenium(idp=None,sp=None,debugSelenium=False): label_sp = get_label(sp) sha1_idp = sha1(idp['entityID']) try: - driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', f'--log-path={e2p.ECCS2SELENIUMLOGDIR}/{sha1_idp}_{label_idp}_{label_sp}.log']) + driver = webdriver.Chrome(e_p.PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', f'--log-path={e_p.ECCS_SELENIUMLOGDIR}/{sha1_idp}_{label_idp}_{label_sp}.log']) except: # Wait 3 seconds before try again to get the webdriver for all kind of exception will occur time.sleep(3) - driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', f'--log-path={e2p.ECCS2SELENIUMLOGDIR}/{sha1_idp}_{label_idp}_{label_sp}.log']) + driver = webdriver.Chrome(e_p.PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', f'--log-path={e_p.ECCS_SELENIUMLOGDIR}/{sha1_idp}_{label_idp}_{label_sp}.log']) else: try: - driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options) + driver = webdriver.Chrome(e_p.PATHCHROMEDRIVER, options=chrome_options) except: # Wait 3 seconds before try again to get the webdriver for all kind of exception will occur time.sleep(3) - driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options) + driver = webdriver.Chrome(e_p.PATHCHROMEDRIVER, options=chrome_options) return driver def follow_all_nested_iframes(driver): @@ -197,7 +197,7 @@ def follow_all_nested_iframes(driver): except NoSuchElementException: return driver.page_source -# ECCS2 Check made by Selenium +# ECCS Check made by Selenium def check_idp_response_selenium(sp,idp,test): # Disable SSL requests warning messages @@ -207,8 +207,8 @@ def check_idp_response_selenium(sp,idp,test): fqdn_idp = get_label(idp['Location']) wayfless_url = f"{sp}{idp['entityID']}" robots = "" - federations_disabled_dict = e2p.FEDS_DISABLED_DICT - idps_disabled_dict = e2p.IDPS_DISABLED_DICT + federations_disabled_dict = e_p.FEDS_DISABLED_DICT + idps_disabled_dict = e_p.IDPS_DISABLED_DICT webdriver_error = 0 # No WebDriver Error # Handle Disabled Idps/Federations @@ -227,13 +227,13 @@ def check_idp_response_selenium(sp,idp,test): # Robots + SSL Check try: hdrs = { - 'User-Agent': f'{e2p.ROBOTS_USER_AGENT}' + 'User-Agent': f'{e_p.ROBOTS_USER_AGENT}' } check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z' - robots = requests.get(f"https://{fqdn_idp}/robots.txt", headers=hdrs, verify=True, timeout=e2p.ECCS2REQUESTSTIMEOUT) + robots = requests.get(f"https://{fqdn_idp}/robots.txt", headers=hdrs, verify=True, timeout=e_p.ECCS_REQUESTSTIMEOUT) if (robots == ""): - robots = requests.get(f"http://{fqdn_idp}/robots.txt", headers=hdrs, verify=False, timeout=e2p.ECCS2REQUESTSTIMEOUT) + robots = requests.get(f"http://{fqdn_idp}/robots.txt", headers=hdrs, verify=False, timeout=e_p.ECCS_REQUESTSTIMEOUT) # Catch SSL Exceptions and block the ECCS check except requests.exceptions.SSLError as e: @@ -258,20 +258,20 @@ def check_idp_response_selenium(sp,idp,test): try: # WebDriver MUST be instanced here to avoid problems with SESSION - driver = get_driver_selenium(idp,sp,e2p.ECCS2SELENIUMDEBUG) + driver = get_driver_selenium(idp,sp,e_p.ECCS_SELENIUMDEBUG) # Exception of WebDriver raises if (driver == None): sys.stderr.write(f"get_driver_selenium() returned None for IDP {idp['entityID']}(SHA1: {sha1(idp['entityID'])}) with SP {get_label(sp)}") return None - driver.set_page_load_timeout(e2p.ECCS2SELENIUMPAGELOADTIMEOUT) - driver.set_script_timeout(e2p.ECCS2SELENIUMSCRIPTTIMEOUT) + driver.set_page_load_timeout(e_p.ECCS_SELENIUMPAGELOADTIMEOUT) + driver.set_script_timeout(e_p.ECCS_SELENIUMSCRIPTTIMEOUT) check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z' driver.get(wayfless_url) - metadata_not_found = re.search(e2p.METADATAPATTERN,driver.page_source, re.I) + metadata_not_found = re.search(e_p.METADATAPATTERN,driver.page_source, re.I) if (metadata_not_found): if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\n[WAYFLESS URL]{wayfless_url} - METADATA NOT FOUND" @@ -284,7 +284,7 @@ def check_idp_response_selenium(sp,idp,test): if ('<iframe' in driver.page_source): follow_all_nested_iframes(driver) - WebDriverWait(driver, e2p.ECCS2SELENIUMPAGELOADTIMEOUT).until( + WebDriverWait(driver, e_p.ECCS_SELENIUMPAGELOADTIMEOUT).until( EC.presence_of_element_located((By.XPATH,'//input[@type="password"]')) ) @@ -296,7 +296,7 @@ def check_idp_response_selenium(sp,idp,test): except TimeoutException as e: - metadata_not_found = re.search(e2p.METADATAPATTERN,driver.page_source, re.I) + metadata_not_found = re.search(e_p.METADATAPATTERN,driver.page_source, re.I) try: input_password_found = driver.find_element(By.XPATH,'//input[@type="password"]') @@ -310,28 +310,28 @@ def check_idp_response_selenium(sp,idp,test): if (stored): return (idp['entityID'],wayfless_url,check_time,"No-eduGAIN-Metadata",webdriver_error) elif(driver.page_source != "<html><head></head><body></body></html>"): - if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\nInvalid-Form: No valid login form found in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds" - else: pgsrc = f"<h1>Invalid Form: no valid login form found in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds</h1><h2>PAGE SOURCE:</h2><br/>{driver.page_source}" + if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\nInvalid-Form: No valid login form found in {e_p.ECCS_SELENIUMPAGELOADTIMEOUT} seconds" + else: pgsrc = f"<h1>Invalid Form: no valid login form found in {e_p.ECCS_SELENIUMPAGELOADTIMEOUT} seconds</h1><h2>PAGE SOURCE:</h2><br/>{driver.page_source}" stored = store_page_source(pgsrc,idp,sp,test) if (stored): return (idp['entityID'],wayfless_url,check_time,"Invalid-Form",webdriver_error) else: - if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\nTimeout: No valid login form loaded in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds." - else: pgsrc = f"<h1>Timeout - No valid login form found in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds.</h1><h2>PAGE SOURCE:</h2><br/>{driver.page_source}" + if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\nTimeout: No valid login form loaded in {e_p.ECCS_SELENIUMPAGELOADTIMEOUT} seconds." + else: pgsrc = f"<h1>Timeout - No valid login form found in {e_p.ECCS_SELENIUMPAGELOADTIMEOUT} seconds.</h1><h2>PAGE SOURCE:</h2><br/>{driver.page_source}" stored = store_page_source(pgsrc,idp,sp,test) if (stored): return (idp['entityID'],wayfless_url,check_time,"Timeout",webdriver_error) except e: - if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\nTimeout: No valid login form loaded in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds." + if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source}\nTimeout: No valid login form loaded in {e_p.ECCS_SELENIUMPAGELOADTIMEOUT} seconds." else: pgsrc = driver.page_source - stored = store_page_source(f"<h1>Timeout - No valid login form found in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds.</h1><br/><p>{pgsrc}</p>",idp,sp,test) + stored = store_page_source(f"<h1>Timeout - No valid login form found in {e_p.ECCS_SELENIUMPAGELOADTIMEOUT} seconds.</h1><br/><p>{pgsrc}</p>",idp,sp,test) if (stored): return (idp['entityID'],wayfless_url,check_time,"Timeout",webdriver_error) # <input type="password"> found - # This IF is for those IdPs that Timeout is caused by an image or other that do now prevent the Login process. + # This IF is for those IdPs that Timeout is caused by an image or other that do not prevent the Login process. if (test): pgsrc = f"\n[PAGE_SOURCE]\n{driver.page_source} - Timeout but OK" else: pgsrc = driver.page_source stored = store_page_source(pgsrc,idp,sp,test) diff --git a/web/details_close_transparent.png.1 b/web/details_close_transparent.png.1 deleted file mode 100644 index 9a942ce400589c1c7b726fc0b99577b45c7b5b3f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 570 zcmeAS@N?(olHy`uVBq!ia0y~yV2}V|4mJh`h6m-gKNuJoSc;uILpV4%IBGajIv5xj zI14-?iy0UgT0xlc(VPDV85kHOOI#yLg7ec#$`gxH8OqDc^)mCai<1)zQuXqS(r3T3 zkz!zA-0tb(7~;{n_S!+eU`G+Q2kW<TS1>ZI%}_mh>`2Rz>kA$jzv$A~)6CDXDRH6O zWA2VlwO?Ef(ua?|IJ7HZ%8np&Cy~O!WuDIObM*`U-<|vUetp`xIiJsm@EkYEOOiYL zg?Ii(&h;Od;y<xySw7~yb67X#fc2iE-$mvh;Eg#PZ{z!Cy|NgO*hlXD8@sLvywkZ> zFyUXMgzCN~<(!8_6R)b6Y@GWq_MEb=+9BhfxlVi94Rh3b8|RuF5!-%F<KAJ5vJb++ ze;bc0JzwtWEw`@2xblYd46&&jqM9oBzkhU3ImXrJ>N=%0?9!g#gV`~Mz6z-;R4tG8 z-TZc%j$Y=rtVdfmTNN<BZAy~ox%OYqDs<v%o6{Oyk9#(1{k!^s_bZQ6p1YWfY;|bO zkvo=0HyO0+-4Tu66Se3^&ZU^9?nRo{yyhgt#xFjoeDU5{p(E)99qfObOB2g(pUW@H z|G+$1y_x$^mic*$lNR&Tdw<$oDpGpG5w6oEx$%>7po#NE!MZvHzU+w7t-{X(mZT;; zt(3J<)%$pR#lyWjzI+d^35xlZoA}jVsQ!P;Igjp)j@4U~gRcG0dD)eI+WVpP%pd%} Y7bFCIU|VyBfq{X+)78&qol`;+0AR)ZH2?qr diff --git a/web/eccs2.css b/web/eccs.css similarity index 100% rename from web/eccs2.css rename to web/eccs.css diff --git a/web/eccs2.js b/web/eccs.js similarity index 96% rename from web/eccs2.js rename to web/eccs.js index 7c9a821..ca0c733 100644 --- a/web/eccs2.js +++ b/web/eccs.js @@ -1,6 +1,6 @@ -// Needed to draw the ECCS2 DataTable +// Needed to draw the ECCS DataTable var table; -var url = "/eccs2/api/eccsresults?eccsdt=1"; +var url = "/eccs/api/eccsresults?eccsdt=1"; var infoCircle = '<a href="https://wiki.geant.org/display/eduGAIN/eduGAIN+Connectivity+Check+2#eduGAINConnectivityCheck2-Statusesandresults"><i class="fas fa-info-circle"></i></a>'; /* @@ -142,12 +142,12 @@ function SHA1(msg) { } -// PHP Variables retrieved from eccs2.php +// PHP Variables retrieved from eccs.php // idp (entityID of the IdP) // date (date time of the check) // reg_auth (the IdP RegistrationAuthority) -// status (the ECCS2 IdP Status) -// check_result (the ECCS2 check result) +// status (the ECCS IdP Status) +// check_result (the ECCS check result) if (date) { url = url.concat("&date=" + date); } @@ -167,7 +167,7 @@ if (check_result) { function getPastResults() { var checkDate = $.datepicker.formatDate("yy-mm-dd", $('#datepicker').datepicker().datepicker('getDate')); - url = "/eccs2/api/eccsresults?eccsdt=1&date=" + checkDate; + url = "/eccs/api/eccsresults?eccsdt=1&date=" + checkDate; $("#datepicker").datepicker("setDate",checkDate); table.ajax.url( url ).load(); @@ -245,7 +245,7 @@ function format ( d ) { '<td>'+d.sp1.checkTime+'</td>'+ '<td>'+getCheckResult(d.sp1.checkResult)+'</td>'+ //'<td>'+d.sp1.httpCode+'</td>'+ - '<td><a href="/eccs2html/'+d.date+'/'+SHA1(d.entityID)+'---'+getHostname(d.sp1.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+ + '<td><a href="/eccs/html/'+d.date+'/'+SHA1(d.entityID)+'---'+getHostname(d.sp1.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+ '<td><a href="'+d.sp1.wayflessUrl+'" target="_blank">Click to retry</a></td>'+ '</tr>'+ '<tr>'+ @@ -254,7 +254,7 @@ function format ( d ) { '<td>'+d.sp2.checkTime+'</td>'+ '<td>'+getCheckResult(d.sp2.checkResult)+'</td>'+ //'<td>'+d.sp2.httpCode+'</td>'+ - '<td><a href="/eccs2html/'+d.date+'/'+SHA1(d.entityID)+'---'+getHostname(d.sp2.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+ + '<td><a href="/eccs/html/'+d.date+'/'+SHA1(d.entityID)+'---'+getHostname(d.sp2.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+ '<td><a href="'+d.sp2.wayflessUrl+'" target="_blank">Click to retry</a></td>'+ '</tr>'+ '</table>'; @@ -313,7 +313,7 @@ $(document).ready(function() { ], "rowCallback": function( row, data, index ) { if (data.status == "ERROR") { - //$('td', row).css('background-color', '#EA4335'); // NEW ECCS2 + //$('td', row).css('background-color', '#EA4335'); // NEW ECCS $('td', row).css('background-color', '#EA3D3F'); // OLD ECCS //$('td', row).css('background-color', '#FF0000'); //$('td', row).css('background-color', '#F22422'); @@ -323,7 +323,7 @@ $(document).ready(function() { } if (data.status == "OK") { //$('td', row).css('background-color', '#34A853'); - //$('td', row).css('background-color', '#00CE00'); // NEW ECCS2 + //$('td', row).css('background-color', '#00CE00'); // NEW ECCS $('td', row).css('background-color', '#72F81B'); // OLD ECCS } }, diff --git a/web/index.php b/web/index.php index 1644096..befae70 100644 --- a/web/index.php +++ b/web/index.php @@ -3,7 +3,7 @@ $directory = "../output"; $files = scandir ($directory); $firstFile = $files[3]; // [0] = '.' ; [1] = '..' ; [2] = '.gitignore' -$str2strip = array("eccs2_", ".log"); +$str2strip = array("eccs_", ".log"); $firstDate = str_replace($str2strip, "", $firstFile); $files = scandir($directory, SCANDIR_SORT_DESCENDING); @@ -28,7 +28,7 @@ $data['check_result'] = htmlspecialchars($_GET["check_result"]); <script type="text/javascript" src="https://cdn.datatables.net/1.10.22/js/jquery.dataTables.min.js"></script> <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css"/> <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css"/> - <link rel="stylesheet" type="text/css" href="eccs2.css" /> + <link rel="stylesheet" type="text/css" href="eccs.css" /> <script> $( function() { $( "#datepicker" ).datepicker({ @@ -39,7 +39,7 @@ $data['check_result'] = htmlspecialchars($_GET["check_result"]); }).datepicker("setDate","<?php echo $data['date'] ?>"); } ); </script> - <title>eduGAIN Connectivity Check Service 2</title> + <title>eduGAIN Connectivity Check Service</title> </head> <body> <div id="status"> @@ -86,6 +86,6 @@ $data['check_result'] = htmlspecialchars($_GET["check_result"]); var status = "<?php echo $data['status'] ?>"; var check_result = "<?php echo $data['check_result'] ?>"; </script> - <script type="text/javascript" src="eccs2.js" /></script> + <script type="text/javascript" src="eccs.js" /></script> </body> </html> -- GitLab