diff --git a/README.md b/README.md index eaeac27d763a5db57dcf0b1841a5ecce414eb85c..5f58d463a51e746e0b24bb4e0722f885bf9c9c7c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# EduGAIN Connectivity Check Service 2 +# EduGAIN Connectivity Check Service 2 - ECCS2 # Requirements Hardware @@ -10,7 +10,7 @@ # Requirements Software * Apache Server + WSGI -* Python 3.8 +* Python 3.8 (tested with v3.8.3) * Selenim + Chromium Web Brower # HOWTO Install and Configure @@ -42,18 +42,17 @@ 7. Create link of Python3.8 for scripts: * `sudo ln -s /usr/local/bin/python3.8 /usr/bin/python3.8` -# Install Apache Web Server + WSGI for ECCS2 API - -* `sudo apt install libapache2-mod-wsgi-py3 python3-dev` -* `sudo a2enmod wsgi` # Install requirements for uWSGI used by ECCS2 API: -* `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python-dev` + +* `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python3-dev` + # Install Chromium used by Selenium * `sudo apt install chromium chromium-l10n git jq` + # Install ECCS2 * `cd ~ ; git clone https://github.com/malavolti/eccs2.git` @@ -66,61 +65,63 @@ # Configure ECCS2 -* `cp eccs2properties.py.template eccs2properties.py` (and change it on your needs) -* `sudo cp eccs2.service /etc/systemd/system/eccs2.service` -* `sudo systemctl daemon-reload` -* `sudo systemctl enable eccs2.service` -* `sudo crontab -u debian -e` +1. Configure ECCS2 properties + * `vim eccs2properties.py` (and change it on your needs) - ```bash - 0 0 * * * /bin/bash /opt/eccs2/cleanAndRunEccs2.sh > /opt/eccs2/logs/eccs2cron.log 2>&1 - ``` +2. Configure ECCS2 cron job for your local user (`debian` into this example): + * `sudo crontab -u debian -e` -# Run ECCS2 - * `./runEccs2.py` or `./cleanAndRunEccs2.py` + ```bash + 0 4 * * * /bin/bash /opt/eccs2/cleanAndRunEccs2.sh > /opt/eccs2/logs/eccs2cron.log 2>&1 + ``` -# API Development Server +3. Configure the ECCS2 systemd service to enable its API: + * `sudo cp eccs2.service /etc/systemd/system/eccs2.service` + * `sudo systemctl daemon-reload` + * `sudo systemctl enable eccs2.service` + * `sudo systemctl start eccs2.service` -* `cd ~/eccs2 ; ./api.py` +4. Configure Apache for the ECCS2 Web side: + * `sudo cp eccs2.conf /etc/apache2/conf-available/eccs2.conf` + * `sudo a2enconf eccs2.conf` + * `sudo systemctl reload apache2.service` -# API -* `/eccs/test` (Trivial Test) -* `/eccs/checks` (Return the results of the last checks) -* `/eccs/checks?<parameter>=<value>`: - * `date=2020-02-20` (select date) - * `idp=Any%20words%20do%20you%20like%20url%20encoded` - * `status=` - * 'OK' - * 'TIMEOUT' - * 'No-eduGAIN-Metadata' - * 'Form-Invalid' - * 'Excluded' -* /eccs/eccsresults (Return the results of the last check ready for ECCS Gui) +# Run ECCS2 manually -# APACHE CONFIGURATION + * `cd ~/eccs2` + * `./cleanAndRunEccs2.py` (to run a full and clean check) + * `./runEccs2.py` (to run a full check on the existing inputs) + * `./runEccs2.py --idp <IDP-ENTITYID>` (to run check on a single IdP) + * `./runEccs2.py --idp --test` (to run a full check on a single IdP without effects) + * `./runEccs2.py --idp <IDP-ENTITYID> --test` (to run check on a single IdP without effects) -* `sudo vim /etc/apache2/sites-availabled/eccs2.conf - ```apache - <IfModule mod_alias.c> - Alias /eccs2 /opt/eccs2/web - Alias /eccs2html /opt/eccs2/html +# ECCS2 API Development Server + +* `cd ~/eccs2 ; ./api.py` - <Directory /opt/eccs2/web> - DirectoryIndex index.php - Require all granted - </Directory> - <Directory /opt/eccs2/html> - Require all granted - </Directory> - </IfModule> - ``` +# ECCS2 API JSON + +* `/api/test` (Trivial Test) +* `/api/eccsresults` (Return the results of the last check ready for ECCS Gui) +* `/api/eccsresults?<parameter>=<value>`: + * `date=2020-02-20` (select date) + * `idp=https://idp.example.org/idp/shibboleth` (select a specific idp) + * `status=` (select specific ECCS2 status) + * 'OK' + * 'ERROR' + * 'DISABLED' + * `reg_auth=https://reg.auth.example.org` (select a specific Registration Authority) +* `/api/fedstats` +* `/api/fedstats?reg_auth=https://reg.auth.example.org`: -* `sudo a2ensite eccs2.conf` -* `sudo systemctl reload apache2.service` # UTILITY FOR WEB INTERFACE The available dates are provided by the first and the last file created into the `output/` directory + +To clean the ECCS2 results from files older than last 7 days use: + +* `clean7daysOlderFiles.sh` diff --git a/api.py b/api.py index 13a395d2208e5c7c0adf989d775a6f216975a3af..6608ae4504521e58ea6e0657c4140ab749b9ab10 100755 --- a/api.py +++ b/api.py @@ -9,8 +9,6 @@ from flask import Flask, request, jsonify from flask_restful import Resource, Api from utils import getLogger, getListFeds, getRegAuthDict -#from eccs2properties import ECCS2FAILEDCMD, ECCS2FAILEDCMDIDP, ECCS2STDOUT, ECCS2STDERR, ECCS2STDOUTIDP, ECCS2STDERRIDP, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE - app = Flask(__name__) api = Api(app) @@ -27,8 +25,8 @@ def buildEmailAddress(listContacts): return hrefList - -def existsInFile(file_path, value, research_item, eccsDataTable): +# Research the value of the research_item into ECCS2 output files +def existsInFile(file_path, value, research_item, eccsDataTable, date): try: with open(file_path,"r",encoding="utf-8") as fo: lines = fo.readlines() @@ -36,7 +34,7 @@ def existsInFile(file_path, value, research_item, eccsDataTable): if (eccsDataTable): return '' else: - return jsonify(error='FileNotFound: ECCS2 script has not been executed on this day') + return jsonify(error='FileNotFound: ECCS2 script has not been executed on %s yet' % date) for line in lines: aux = json.loads(line) @@ -51,13 +49,13 @@ def existsInFile(file_path, value, research_item, eccsDataTable): ### Classes -# Test +# /api/test class Test(Resource): def get(self): return {'test':'It Works!'} -# /eccs2/api/eccsresults +# /api/eccsresults class EccsResults(Resource): def get(self): @@ -79,11 +77,11 @@ class EccsResults(Resource): return jsonify(error="Incorrect status provided. It can be 'ok','disabled','error'") if 'idp' in request.args: idp = request.args['idp'] - if (not existsInFile(file_path, idp, "entityID", eccsDataTable)): + if (not existsInFile(file_path, idp, "entityID", eccsDataTable, date)): return jsonify(error="Identity Provider not found with the entityID: %s" % idp) if 'reg_auth' in request.args: reg_auth = request.args['reg_auth'] - if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable)): + if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable, date)): return jsonify(error="Identity Providers not found with the Registration Authority: %s" % reg_auth) lines = [] @@ -136,7 +134,7 @@ class EccsResults(Resource): return jsonify(results) -# /eccs2/api/fedstats +# /api/fedstats class FedStats(Resource): def get(self): list_feds = getListFeds(ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE) @@ -152,7 +150,7 @@ class FedStats(Resource): file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,date) if ('reg_auth' in request.args): reg_auth = request.args['reg_auth'] - if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable)): + if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable, date)): return jsonify(error="Registration Authority not found") lines = [] @@ -165,7 +163,7 @@ class FedStats(Resource): if (eccsDataTable): return '' else: - return jsonify(error='FileNotFound: ECCS2 script has not been executed on this day') + return jsonify(error='FileNotFound: ECCS2 script has not been executed on %s yet' % date) if (reg_auth): resultDict = {'date': date, 'registrationAuthority': reg_auth, 'OK': 0, 'ERROR': 0, 'DISABLED': 0} @@ -217,6 +215,7 @@ api.add_resource(FedStats, '/fedstats') # Route_3 if __name__ == '__main__': + # Useful only for API development Server #app.config['JSON_AS_ASCII'] = True #app.logger.removeHandler(default_handler) #app.logger = getLogger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO") diff --git a/clean7daysOldfiles.sh b/clean7daysOldfiles.sh new file mode 100755 index 0000000000000000000000000000000000000000..4e43adf05a2d7ddbdd5886777df6062a2fca1641 --- /dev/null +++ b/clean7daysOldfiles.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Remove ECCS2 result older than 7 days +find /opt/eccs2/output/* -mtime +6 -type f -delete + +# Remove ECCS2 logs older than 7 days +find /opt/eccs2/logs/* -mtime +6 -type f -delete + +# Remove ECCS2 HTML code older than 7 days +find /opt/eccs2/html/* -mtime +6 -type f -delete diff --git a/eccs2.conf b/eccs2.conf new file mode 100644 index 0000000000000000000000000000000000000000..e67fe72402bb3780bc3071f9e264defded6ef680 --- /dev/null +++ b/eccs2.conf @@ -0,0 +1,17 @@ +<IfModule mod_alias.c> + Alias /eccs2 /opt/eccs2/web + Alias /eccs2html /opt/eccs2/html + + <Directory /opt/eccs2/web> + DirectoryIndex index.php + Require all granted + </Directory> + + <Directory /opt/eccs2/html> + Require all granted + </Directory> +</IfModule> + +<IfModule mod_proxy_uwsgi.c> + ProxyPass /eccs2/api uwsgi://127.0.0.1:8000/ +</IfModule> diff --git a/eccs2.ini b/eccs2.ini new file mode 100644 index 0000000000000000000000000000000000000000..2464b517fa9dae7160e090fa1bb1ad7d2691390f --- /dev/null +++ b/eccs2.ini @@ -0,0 +1,38 @@ +[uwsgi] +project = eccs2 +base = /opt + +chdir = %(base)/%(project) + +master = true +processes = 2 + +uid = debian +gid = debian + +socket = 127.0.0.1:8000 +chmod-socket = 660 +vacuum = true + +die-on-term = true + +# set max connections to 1024 in uWSGI +listen = 128 + +virtualenv = eccs2venv +wsgi-file = eccs2wsgi.py + +logto = logs/%(project)uwsgi.log +log-maxsize = 100000000 +log-backupname = logs/%(project)uwsgi.old.log + +# Ogni 1000 request riavvia i workers +max-requests = 1000 + +# respawn processes taking more than takes more then ... seconds +harakiri = 20 + +pidfile = %(project).pid +touch-reload = %(project).ini +#stats = 127.0.0.1:9193 +#stats-http = True diff --git a/eccs2.service b/eccs2.service new file mode 100644 index 0000000000000000000000000000000000000000..80bdf430e35e9ce0786dd4afbf3303ff4a4795d5 --- /dev/null +++ b/eccs2.service @@ -0,0 +1,19 @@ +[Install] +WantedBy=multi-user.target + +[Unit] +Description=uWSGI server for ECCS2 +After=network.target + +[Service] +User=debian +Group=debian +WorkingDirectory=/opt/eccs2 +RuntimeDirectory=/opt/eccs2 +Restart=always +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all +#Environment="PATH=/opt/eccs2/eccs2venv/bin" +ExecStart=/opt/eccs2/eccs2venv/bin/uwsgi --ini /opt/eccs2/eccs2.ini diff --git a/eccs2wsgi.py b/eccs2wsgi.py new file mode 100644 index 0000000000000000000000000000000000000000..c6dc305321fee898fe3cf1c0f1baffd56eb8015d --- /dev/null +++ b/eccs2wsgi.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3.8 + +from api import app as application + +if __name__ == "__main__": + application.run()