Fixed API and enriched README file

README.md

-# EduGAIN Connectivity Check Service 2
+# EduGAIN Connectivity Check Service 2 - ECCS2
 
 # Requirements Hardware
 
@@ -10,7 +10,7 @@
 # Requirements Software
 
 * Apache Server + WSGI
-* Python 3.8
+* Python 3.8 (tested with v3.8.3)
 * Selenim + Chromium Web Brower
 
 # HOWTO Install and Configure
@@ -42,18 +42,17 @@
 7. Create link of Python3.8 for scripts:
    * `sudo ln -s /usr/local/bin/python3.8 /usr/bin/python3.8`
 
-# Install Apache Web Server + WSGI for ECCS2 API
-
-* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
-* `sudo a2enmod wsgi`
 
 # Install requirements for uWSGI used by ECCS2 API:
-* `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python-dev`
+
+* `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python3-dev`
+
 
 # Install Chromium used by Selenium
 
 * `sudo apt install chromium chromium-l10n git jq`
 
+
 # Install ECCS2
 
 * `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
@@ -66,61 +65,63 @@
 
 # Configure ECCS2
 
-* `cp eccs2properties.py.template eccs2properties.py` (and change it on your needs)
-* `sudo cp eccs2.service /etc/systemd/system/eccs2.service`
-* `sudo systemctl daemon-reload`
-* `sudo systemctl enable eccs2.service`
-* `sudo crontab -u debian -e`
+1. Configure ECCS2 properties
+   * `vim eccs2properties.py` (and change it on your needs)
 
-  ```bash
-  0 0 * * * /bin/bash /opt/eccs2/cleanAndRunEccs2.sh > /opt/eccs2/logs/eccs2cron.log 2>&1  
-  ```
+2. Configure ECCS2 cron job for your local user (`debian` into this example):
+   * `sudo crontab -u debian -e`
 
-# Run ECCS2
-  * `./runEccs2.py` or `./cleanAndRunEccs2.py`
+     ```bash
+     0 4 * * * /bin/bash /opt/eccs2/cleanAndRunEccs2.sh > /opt/eccs2/logs/eccs2cron.log 2>&1  
+     ```
 
-# API Development Server
+3. Configure the ECCS2 systemd service to enable its API:
+   * `sudo cp eccs2.service /etc/systemd/system/eccs2.service`
+   * `sudo systemctl daemon-reload`
+   * `sudo systemctl enable eccs2.service`
+   * `sudo systemctl start eccs2.service`
 
-* `cd ~/eccs2 ; ./api.py`
+4. Configure Apache for the ECCS2 Web side:
+   * `sudo cp eccs2.conf /etc/apache2/conf-available/eccs2.conf`
+   * `sudo a2enconf eccs2.conf`
+   * `sudo systemctl reload apache2.service`
 
-# API
 
-* `/eccs/test` (Trivial Test)
-* `/eccs/checks` (Return the results of the last checks)
-* `/eccs/checks?<parameter>=<value>`:
-  * `date=2020-02-20` (select date)
-  * `idp=Any%20words%20do%20you%20like%20url%20encoded`
-  * `status=`
-    * 'OK'
-    * 'TIMEOUT'
-    * 'No-eduGAIN-Metadata'
-    * 'Form-Invalid'
-    * 'Excluded'
-* /eccs/eccsresults (Return the results of the last check ready for ECCS Gui)
+# Run ECCS2 manually
 
-# APACHE CONFIGURATION
+  * `cd ~/eccs2`
+  * `./cleanAndRunEccs2.py` (to run a full and clean check)
+  * `./runEccs2.py` (to run a full check on the existing inputs)
+  * `./runEccs2.py --idp <IDP-ENTITYID>` (to run check on a single IdP)
+  * `./runEccs2.py --idp --test` (to run a full check on a single IdP without effects)
+  * `./runEccs2.py --idp <IDP-ENTITYID> --test` (to run check on a single IdP without effects)
 
-* `sudo vim /etc/apache2/sites-availabled/eccs2.conf
 
-  ```apache
-  <IfModule mod_alias.c>
-     Alias /eccs2 /opt/eccs2/web
-     Alias /eccs2html /opt/eccs2/html
+# ECCS2 API Development Server
+
+* `cd ~/eccs2 ; ./api.py`
 
-     <Directory /opt/eccs2/web>
-        DirectoryIndex index.php
-        Require all granted
-     </Directory>
 
-     <Directory /opt/eccs2/html>
-        Require all granted
-     </Directory>
-  </IfModule>
-  ```
+# ECCS2 API JSON
+
+* `/api/test` (Trivial Test)
+* `/api/eccsresults` (Return the results of the last check ready for ECCS Gui)
+* `/api/eccsresults?<parameter>=<value>`:
+  * `date=2020-02-20` (select date)
+  * `idp=https://idp.example.org/idp/shibboleth`  (select a specific idp)
+  * `status=` (select specific ECCS2 status)
+    * 'OK'
+    * 'ERROR'
+    * 'DISABLED'
+  * `reg_auth=https://reg.auth.example.org` (select a specific Registration Authority)
+* `/api/fedstats`
+* `/api/fedstats?reg_auth=https://reg.auth.example.org`:
 
-* `sudo a2ensite eccs2.conf`
-* `sudo systemctl reload apache2.service`
 
 # UTILITY FOR WEB INTERFACE
 
 The available dates are provided by the first and the last file created into the `output/` directory
+
+To clean the ECCS2 results from files older than last 7 days use:
+
+* `clean7daysOlderFiles.sh`

api.py

@@ -9,8 +9,6 @@ from flask import Flask, request, jsonify
 from flask_restful import Resource, Api
 from utils import getLogger, getListFeds, getRegAuthDict
 
-#from eccs2properties import ECCS2FAILEDCMD, ECCS2FAILEDCMDIDP, ECCS2STDOUT, ECCS2STDERR, ECCS2STDOUTIDP, ECCS2STDERRIDP, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE
-
 app = Flask(__name__)
 api = Api(app)
 
@@ -27,8 +25,8 @@ def buildEmailAddress(listContacts):
  
     return hrefList
 
-
-def existsInFile(file_path, value, research_item, eccsDataTable):
+# Research the value of the research_item into ECCS2 output files
+def existsInFile(file_path, value, research_item, eccsDataTable, date):
     try:
        with open(file_path,"r",encoding="utf-8") as fo:
             lines = fo.readlines()
@@ -36,7 +34,7 @@ def existsInFile(file_path, value, research_item, eccsDataTable):
         if (eccsDataTable):
            return ''
         else:
-           return jsonify(error='FileNotFound: ECCS2 script has not been executed on this day')
+           return jsonify(error='FileNotFound: ECCS2 script has not been executed on %s yet' % date)
 
     for line in lines:
         aux = json.loads(line)
@@ -51,13 +49,13 @@ def existsInFile(file_path, value, research_item, eccsDataTable):
 
 ### Classes
 
-# Test
+# /api/test
 class Test(Resource):
     def get(self):
         return {'test':'It Works!'}
 
 
-# /eccs2/api/eccsresults
+# /api/eccsresults
 class EccsResults(Resource):
     def get(self):
 
@@ -79,11 +77,11 @@ class EccsResults(Resource):
               return jsonify(error="Incorrect status provided. It can be 'ok','disabled','error'")
        if 'idp' in request.args:
           idp = request.args['idp']
-          if (not existsInFile(file_path, idp, "entityID", eccsDataTable)):
+          if (not existsInFile(file_path, idp, "entityID", eccsDataTable, date)):
              return jsonify(error="Identity Provider not found with the entityID: %s" % idp)
        if 'reg_auth' in request.args:
           reg_auth = request.args['reg_auth']
-          if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable)):
+          if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable, date)):
              return jsonify(error="Identity Providers not found with the Registration Authority: %s" % reg_auth)
 
        lines = []
@@ -136,7 +134,7 @@ class EccsResults(Resource):
        return jsonify(results)
 
 
-# /eccs2/api/fedstats
+# /api/fedstats
 class FedStats(Resource):
    def get(self):
        list_feds = getListFeds(ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE)
@@ -152,7 +150,7 @@ class FedStats(Resource):
           file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,date)
        if ('reg_auth' in request.args):
           reg_auth = request.args['reg_auth']
-          if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable)):
+          if (not existsInFile(file_path, reg_auth, "registrationAuthority", eccsDataTable, date)):
              return jsonify(error="Registration Authority not found")
 
        lines = []
@@ -165,7 +163,7 @@ class FedStats(Resource):
            if (eccsDataTable):
               return ''
            else:
-              return jsonify(error='FileNotFound: ECCS2 script has not been executed on this day')
+              return jsonify(error='FileNotFound: ECCS2 script has not been executed on %s yet' % date)
 
        if (reg_auth):
           resultDict = {'date': date, 'registrationAuthority': reg_auth, 'OK': 0, 'ERROR': 0, 'DISABLED': 0}
@@ -217,6 +215,7 @@ api.add_resource(FedStats, '/fedstats') # Route_3
 
 if __name__ == '__main__':
 
+   # Useful only for API development Server
    #app.config['JSON_AS_ASCII'] = True
    #app.logger.removeHandler(default_handler)
    #app.logger = getLogger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO")

clean7daysOldfiles.sh

+#!/bin/bash
+
+# Remove ECCS2 result older than 7 days
+find /opt/eccs2/output/* -mtime +6 -type f -delete
+
+# Remove ECCS2 logs older than 7 days
+find /opt/eccs2/logs/* -mtime +6 -type f -delete
+
+# Remove ECCS2 HTML code older than 7 days
+find /opt/eccs2/html/* -mtime +6 -type f -delete

eccs2.conf

+<IfModule mod_alias.c>
+    Alias /eccs2 /opt/eccs2/web
+    Alias /eccs2html /opt/eccs2/html
+
+    <Directory /opt/eccs2/web>
+       DirectoryIndex index.php
+       Require all granted
+    </Directory>
+
+    <Directory /opt/eccs2/html>
+       Require all granted
+    </Directory>
+</IfModule>
+
+<IfModule mod_proxy_uwsgi.c>
+    ProxyPass /eccs2/api uwsgi://127.0.0.1:8000/
+</IfModule>

eccs2.ini

+[uwsgi]
+project        = eccs2
+base           = /opt
+
+chdir          = %(base)/%(project)
+
+master         = true
+processes      = 2
+
+uid            = debian
+gid            = debian
+
+socket         = 127.0.0.1:8000
+chmod-socket   = 660
+vacuum         = true
+
+die-on-term    = true
+
+# set max connections to 1024 in uWSGI
+listen         = 128
+
+virtualenv     = eccs2venv
+wsgi-file      = eccs2wsgi.py
+
+logto          = logs/%(project)uwsgi.log
+log-maxsize    = 100000000
+log-backupname = logs/%(project)uwsgi.old.log
+
+# Ogni 1000 request riavvia i workers
+max-requests   = 1000
+
+# respawn processes taking more than takes more then ... seconds
+harakiri       = 20
+
+pidfile        = %(project).pid
+touch-reload   = %(project).ini
+#stats         = 127.0.0.1:9193
+#stats-http    = True

eccs2.service

+[Install]
+WantedBy=multi-user.target
+
+[Unit]
+Description=uWSGI server for ECCS2
+After=network.target
+
+[Service]
+User=debian
+Group=debian
+WorkingDirectory=/opt/eccs2
+RuntimeDirectory=/opt/eccs2
+Restart=always
+KillSignal=SIGQUIT
+Type=notify
+StandardError=syslog
+NotifyAccess=all
+#Environment="PATH=/opt/eccs2/eccs2venv/bin"
+ExecStart=/opt/eccs2/eccs2venv/bin/uwsgi --ini /opt/eccs2/eccs2.ini

eccs2wsgi.py

+#!/usr/bin/env python3.8
+
+from api import app as application
+
+if __name__ == "__main__":
+    application.run()