diff --git a/.gitignore b/.gitignore index 918c66b23d8b68c4f8ad7a81c959f579996d5530..febb0efbc9cc21cb6456816fc21c5356b51f71fb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ *.pyc *eccs2venv* eccs2.pid +chromedriver +python* +eccs2.ini +eccs2.service diff --git a/README.md b/README.md index 5800e288f6b6717773c05d473a75bbfb56270deb..1f6fbd85e5e339b2af8b41da724de4f49a1a1320 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,15 @@ + [Python 3.8](#python-38) 9. [Install Selenium and Chromedriver](#install-selenium-and-chromedriver) 10. [Install Chromium needed by Selenium](#install-chromium-needed-by-selenium) -11. [ECCS2](#eccs2) +11. [ECCS2 Script](#eccs2-script) * [Install](#install) * [Configure](#configure) - * [Execute](#execute) -12. [Install requirements for uWSGI used by ECCS2 API](#install-requirements-for-uwsgi-used-by-eccs2-api) -13. [ECCS2 API Development Server](#eccs2-api-development-server) -14. [ECCS2 API JSON](#eccs2-api-json) -15. [Utility for web interface](#utility-for-web-interface) -16. [Authors](#authors) + * [Execute](#execute) +12. [ECCS2 API JSON](#eccs2-api-json) +13. [Utility for web interface](#utility-for-web-interface) +14. [Utility for developers](#utility-for-developers) + * [ECCS2 API Development Server](#eccs2-api-development-server) +15. [Authors](#authors) # Introduction @@ -78,16 +78,20 @@ The tool uses following status for IdPs: * OS: Debian 9, CentOS 7.8 (tested) * HDD: 10 GB * RAM: 4 GB -* CPU: >= 2 vCPU +* CPU: >= 2 vCPU (suggested) # Requirements Software * Apache Server + WSGI -* Python 3.8 (tested with v3.8.3) +* Python 3.8 (tested with v3.8.3,v3.8.5) * Selenim + Chromium Web Brower # HOWTO Install and Configure +## Download ECCS2 Repository + +* `cd $HOME ; git clone https://github.com/malavolti/eccs2.git` + ## Install Python 3.8.x ### CentOS 7 requirements @@ -111,24 +115,26 @@ The tool uses following status for IdPs: ### Python 3.8 -1. Download the last version of Python 3.8.x from https://www.python.org/downloads/source/: - * `sudo wget https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz -O /usr/local/src/Python-3.8.3.tar.gz` +1. Download the last version of Python 3.8.x from https://www.python.org/downloads/source/ into your home: + * `wget https://www.python.org/ftp/python/3.8.5/Python-3.8.5.tgz -O $HOME/eccs2/Python-3.8.5.tgz` 2. Extract Python source package: - * `cd /usr/local/src/` - * `sudo tar xzf Python-3.8.3.tar.gz` + * `cd $HOME/eccs2/` + * `tar xzf Python-3.8.5.tgz` 3. Build Python from the source package: - * `cd /usr/local/src/Python-3.8.3` - * `sudo ./configure --enable-optimizations` - * `sudo make -j 4` - -4. Install Python 3.8.x (without replacing the system `python3` command) under `/usr/local/bin/python3.8`: - * `sudo make altinstall` - * `python3.8 --version` - -5. Create link of Python3.8 for scripts: - * `sudo ln -s /usr/local/bin/python3.8 /usr/bin/python3.8` + * `cd $HOME/eccs2/Python-3.8.5` + * `./configure --prefix=$HOME/eccs2/python` + * `make` + +4. Install Python 3.8.x under `$HOME/eccs2/python`: + * `make install` + * `$HOME/eccs2/python/bin/python3.8 --version` + + This will install python under your $HOME directory. + +5. Remove useless things: + * `rm -Rf $HOME/eccs2/Python-3.8.5 $HOME/eccs2/Python-3.8.5.tgz` # Install Chromium needed by Selenium @@ -141,11 +147,13 @@ The tool uses following status for IdPs: # Install Selenium and Chromedriver -* `sudo python3.8 -m pip install --upgrade pip` -* `sudo python3.8 -m pip install selenium virtualenv` -* `sudo wget https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_linux64.zip -O /usr/local/src/chromedriver_linux64.zip` -* `cd /usr/bin/` -* `sudo unzip /usr/local/src/chromedriver_linux64.zip` +* `cd $HOME/eccs2/python/bin +* `./python3.8 -m pip install --upgrade pip` +* `./python3.8 -m pip install selenium` +* `wget https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_linux64.zip -O $HOME/eccs2/chromedriver_linux64.zip` +* `cd $HOME/eccs2` +* `unzip chromedriver_linux64.zip` +* `rm chromedriver_linux64.zip` Note: Pay attetion on the chromedriver version: * Debian 9 (stretch): @@ -153,50 +161,34 @@ Note: Pay attetion on the chromedriver version: * CentOS 7.8: * `chromium-browser -version` => Chromium 83.0.4103.116 => https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_linux64.zip - -# ECCS2 +# ECCS2 Script ## Install -* `cd /opt ; git clone https://github.com/malavolti/eccs2.git` -* `cd eccs2` -* `virtualenv --python=/usr/bin/python3.8 eccs2venv` +* `cd $HOME/eccs2` +* `./python/bin/python3.8 -m pip install virtualenv` +* `$HOME/eccs2/python/bin/virtualenv --python=$HOME/eccs2/python/bin/python3.8 eccs2venv` * `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv) - * `python3.8 -m pip install --upgrade pip uwsgi` - * `python3.8 -m pip install -r requirements.txt` + * `python -m pip install -r requirements.txt` ## Configure 1. Configure ECCS2 properties: * `vim eccs2properties.py` (and change it upon your needs) -2. Configure ECCS2 cron job for your local user (`debian` into this example): - * `sudo crontab -u debian -e` (Debian) or `sudo crontab -u centos -e` (CentOS) +2. Add to `PATH` the virtualenv Python `bin` dir: + * `export PATH=${HOME}/eccs2/eccs2venv/bin:${PATH}` + +3. Configure ECCS2 cron job for the local user: + * `crontab -e` ```bash - 0 4 * * * /bin/bash /opt/eccs2/cleanAndRunEccs2.sh > /opt/eccs2/logs/eccs2cron.log 2>&1 + 0 4 * * * /bin/bash $HOME/eccs2/cleanAndRunEccs2.sh > $HOME/eccs2/logs/eccs2cron.log 2>&1 ``` -3. Configure the ECCS2 systemd service to enable its API: - * `vim /opt/eccs2/eccs2.ini` (and change the "`User`" and the "`Group`" values) - * `vim /opt/eccs2/eccs2.service` (and change the "`User`" and the "`Group`" values) - * `sudo cp /opt/eccs2/eccs2.service /etc/systemd/system/eccs2.service` - * `sudo systemctl daemon-reload` - * `sudo systemctl enable eccs2.service` - * `sudo systemctl start eccs2.service` - -4. Configure Apache for the ECCS2 Web side: - * Debian: - * `sudo cp /opt/eccs2/eccs2-debian.conf /etc/apache2/conf-available/eccs2.conf` - * `sudo a2enconf eccs2.conf` - * `sudo systemctl restart apache2.service` - * CentOS: - * `sudo cp /opt/eccs2/eccs2-centos.conf /etc/httpd/conf.d/eccs2.conf` - * `sudo systemctl restart httpd.service` - ## Execute - * `cd /opt/eccs2` + * `cd $HOME/eccs2` * `./cleanAndRunEccs2.py` (to run a full and clean check) * `./runEccs2.py` (to run a full check on the existing inputs) * `./runEccs2.py --idp <IDP-ENTITYID>` (to run check on a single IdP) @@ -204,28 +196,49 @@ Note: Pay attetion on the chromedriver version: * `./runEccs2.py --idp <IDP-ENTITYID> --test` (to run check on a single IdP without effects) The check will run a second time for those IdPs that failed the first execution of the script. - If something prevent the good execution of the ECCS2's check, the `/opt/eccs2/logs/failed-cmd.sh` file will be not empty at the end of the execution. + If something prevent the good execution of the ECCS2's check, the `logs/failed-cmd.sh` file will be not empty at the end of the execution. The "--test" parameter will not change the result of ECCS2, but will write the output on the `logs/stdout_idp_YYYY-MM-DD.log`,`logs/stderr_idp_YYYY-MM-DD.log` and `logs/failed-cmd-idp.sh` files. -# Install requirements for uWSGI used by ECCS2 API +# ECCS2 API Server (uWSGI) -* Debian: - * `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python3-dev unzip` -* CentOS: - * `sudo yum install mod_proxy_uwsgi unzip` - * Enable ECCS2 for SELinux: - * `sudo semanage fcontext -a -t httpd_sys_content_t "/opt/eccs2(/.*)?"` - * `sudo restorecon -R -e /opt/eccs2/` +## Install -# ECCS2 API Development Server +1. Install requirements: + * Debian: + * `sudo apt-get install libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essentials python3-dev unzip` + * CentOS: + * `sudo yum install mod_proxy_uwsgi unzip` + * Configure SElinux to enable ECCS2: + * `sudo semanage fcontext -a -t httpd_sys_content_t "$HOME/eccs2(/.*)?"` + * `sudo restorecon -R -e $HOME/eccs2/` + * `sudo setsebool -P httpd_can_network_connect 1` + +2. Add the systemd service to enable ECCS2 API: + * `cd $HOME/eccs2` + * `cp eccs2.ini.template eccs2.ini + * `cp eccs2.service.template eccs2.service + * `vim eccs2.ini` (and change "`uid`", "`gid`" and "`base`" values opportunely) + * `vim eccs2.service` (and change "`User`","`Group`","`WorkingDirectory`","`RuntimeDirectory`","`ExecStart`" values opportunely) + * `sudo cp $HOME/eccs2/eccs2.service /etc/systemd/system/eccs2.service` + * `sudo systemctl daemon-reload` + * `sudo systemctl enable eccs2.service` + * `sudo systemctl start eccs2.service` -* `cd /opt/eccs2 ; ./api.py` +3. Configure Apache for ECCS2 web side: + * Debian: + * `sudo cp $HOME/eccs2/eccs2-debian.conf /etc/apache2/conf-available/eccs2.conf` + * `sudo a2enconf eccs2.conf` + * `sudo chgrp www-data $HOME ; sudo chmod g+rx $HOME` (Apache needs permission to access the $HOME dir) + * `sudo systemctl restart apache2.service` + * CentOS: + * `sudo cp $HOME/eccs2/eccs2-centos.conf /etc/httpd/conf.d/eccs2.conf` + * `sudo chgrp apache $HOME ; sudo apache g+rx $HOME` (Apache needs permission to access the $HOME dir) + * `sudo systemctl restart httpd.service` # ECCS2 API JSON -* `/api/test` (Trivial Test) -* `/api/eccsresults` (Return the results of the last check ready for ECCS Gui) +* `/api/eccsresults` (Return the results of the last check ready for ECCS web interface) * `/api/eccsresults?<parameter1>=<value1>&<parameter2>=<value2>`: * `date=2020-02-20` (select date) * `idp=https://idp.example.org/idp/shibboleth` (select a specific idp) @@ -237,15 +250,20 @@ Note: Pay attetion on the chromedriver version: * `/api/fedstats` * `/api/fedstats?reg_auth=https://reg.auth.example.org`: - # Utility for web interface The available dates are provided by the first and the last file created into the `output/` directory -To clean the ECCS2 results from files older than last 7 days use: +To clean the ECCS2 results from files older than last 7 days use (modify it on your needs): * `clean7daysOldFiles.sh` +# Utility for developers + +## ECCS2 API Development Server + +* `cd $HOME/eccs2 ; ./api.py` + # Authors ## Original Author diff --git a/api.py b/api.py index 39e7f9b4c5019975995ddf857d377888e88d87bb..deaa70db3a3106dcb4a0c71b642075fd48786abc 100755 --- a/api.py +++ b/api.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3 import json import logging diff --git a/clean7daysOldfiles.sh b/clean7daysOldfiles.sh index 4e43adf05a2d7ddbdd5886777df6062a2fca1641..13d5ba58aa9f61a40395555e19d4eb8e9de90f30 100755 --- a/clean7daysOldfiles.sh +++ b/clean7daysOldfiles.sh @@ -1,10 +1,10 @@ #!/bin/bash # Remove ECCS2 result older than 7 days -find /opt/eccs2/output/* -mtime +6 -type f -delete +find $HOME/eccs2/output/* -mtime +6 -type f -delete # Remove ECCS2 logs older than 7 days -find /opt/eccs2/logs/* -mtime +6 -type f -delete +find $HOME/eccs2/logs/* -mtime +6 -type f -delete # Remove ECCS2 HTML code older than 7 days -find /opt/eccs2/html/* -mtime +6 -type f -delete +find $HOME/eccs2/html/* -mtime +6 -type f -delete diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh index 3d9cca6040e83f866d3fb10f94ff211578d138d5..716d3b063dc4c34193e0114bd5166a1271a834b4 100755 --- a/cleanAndRunEccs2.sh +++ b/cleanAndRunEccs2.sh @@ -2,21 +2,23 @@ # logs/stderr_$date.log is kept to see which IdP had been errors +BASEDIR=$HOME + # Remove old IdP and Fed List -rm -f /opt/eccs2/input/*.json +rm -f $BASEDIR/eccs2/input/*.json # Run ECCS2 -/opt/eccs2/runEccs2.py +$BASEDIR/eccs2/runEccs2.py # Run Failed Command again -bash /opt/eccs2/logs/failed-cmd.sh +bash $BASEDIR/eccs2/logs/failed-cmd.sh # Remove "failed-cmd" and "stdout*" "stderr*" if empty date=$(date '+%Y-%m-%d') -file="/opt/eccs2/logs/failed-cmd.sh" -prefix="/opt/eccs2/eccs2.py '" +file="$BASEDIR/eccs2/logs/failed-cmd.sh" +prefix="$BASEDIR/eccs2/eccs2.py '" suffix="'" -eccs2output="/opt/eccs2/output/eccs2_$date.log" +eccs2output="$BASEDIR/eccs2/output/eccs2_$date.log" declare -a eccs2cmdToRemoveArray while IFS= read -r line diff --git a/eccs2-centos.conf b/eccs2-centos.conf index 57a24c89c46c23f38988d778f8b623d8cf1400b8..4277791a28c3c1a4aef0e33a230b91af8491a80a 100644 --- a/eccs2-centos.conf +++ b/eccs2-centos.conf @@ -1,13 +1,13 @@ <IfModule mod_alias.c> - Alias /eccs2 /opt/eccs2/web - Alias /eccs2html /opt/eccs2/html + Alias /eccs2 /home/<USER>/eccs2/web + Alias /eccs2html /home/<USER>/eccs2/html - <Directory /opt/eccs2/web> + <Directory /home/<USER>/eccs2/web> DirectoryIndex index.php Require all granted </Directory> - <Directory /opt/eccs2/html> + <Directory /home/<USER>/eccs2/html> Require all granted </Directory> </IfModule> diff --git a/eccs2-debian.conf b/eccs2-debian.conf index e67fe72402bb3780bc3071f9e264defded6ef680..539e9a013feb39a89d6b37a959f86f995dd06e85 100644 --- a/eccs2-debian.conf +++ b/eccs2-debian.conf @@ -1,13 +1,13 @@ <IfModule mod_alias.c> - Alias /eccs2 /opt/eccs2/web - Alias /eccs2html /opt/eccs2/html + Alias /eccs2 /home/<USER>/eccs2/web + Alias /eccs2html /home/<USER>/eccs2/html - <Directory /opt/eccs2/web> + <Directory /home/<USER>/eccs2/web> DirectoryIndex index.php Require all granted </Directory> - <Directory /opt/eccs2/html> + <Directory /home/<USER>/eccs2/html> Require all granted </Directory> </IfModule> diff --git a/eccs2.ini b/eccs2.ini.template similarity index 90% rename from eccs2.ini rename to eccs2.ini.template index 2464b517fa9dae7160e090fa1bb1ad7d2691390f..6173c44b1c60a7670425d5ffb6133acf57d86ba9 100644 --- a/eccs2.ini +++ b/eccs2.ini.template @@ -1,14 +1,14 @@ [uwsgi] project = eccs2 -base = /opt +base = /home/<USER> chdir = %(base)/%(project) master = true processes = 2 -uid = debian -gid = debian +uid = <USER> +gid = <USER> socket = 127.0.0.1:8000 chmod-socket = 660 diff --git a/eccs2.py b/eccs2.py index 6be73b40b23519c494896a2e2b3a2dee963aaa74..56ac10fe148144627bc5067bc15a34aaff8b85b6 100755 --- a/eccs2.py +++ b/eccs2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3 import argparse import datetime diff --git a/eccs2.service b/eccs2.service deleted file mode 100644 index 80bdf430e35e9ce0786dd4afbf3303ff4a4795d5..0000000000000000000000000000000000000000 --- a/eccs2.service +++ /dev/null @@ -1,19 +0,0 @@ -[Install] -WantedBy=multi-user.target - -[Unit] -Description=uWSGI server for ECCS2 -After=network.target - -[Service] -User=debian -Group=debian -WorkingDirectory=/opt/eccs2 -RuntimeDirectory=/opt/eccs2 -Restart=always -KillSignal=SIGQUIT -Type=notify -StandardError=syslog -NotifyAccess=all -#Environment="PATH=/opt/eccs2/eccs2venv/bin" -ExecStart=/opt/eccs2/eccs2venv/bin/uwsgi --ini /opt/eccs2/eccs2.ini diff --git a/eccs2.service.template b/eccs2.service.template new file mode 100644 index 0000000000000000000000000000000000000000..5938658a85bdbde7c7fac1d69d5565f2defd15ea --- /dev/null +++ b/eccs2.service.template @@ -0,0 +1,19 @@ +[Install] +WantedBy=multi-user.target + +[Unit] +Description=uWSGI server for ECCS2 +After=network.target + +[Service] +User=<USER> +Group=<USER> +WorkingDirectory=/home/<USER>/eccs2 +RuntimeDirectory=/home/<USER>/eccs2 +Restart=always +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all +#Environment="PATH=/home/<USER>/eccs2/eccs2venv/bin" +ExecStart=/home/<USER>/eccs2/eccs2venv/bin/uwsgi --ini /home/<USER>/eccs2/eccs2.ini diff --git a/eccs2properties.py b/eccs2properties.py index 6e72059269af9fc99c80e789dc7493f7aaf7b058..380e9e70f02e5d262c00da4548197eeb5dafd758 100644 --- a/eccs2properties.py +++ b/eccs2properties.py @@ -1,10 +1,10 @@ -#!/usr/bin/env python3.8 - from datetime import date DAY = date.today().isoformat() -ECCS2DIR = "/opt/eccs2" +ECCS2DIR = "/home/marco/eccs2" +PATHCHROMEDRIVER = "/home/marco/eccs2/chromedriver" +ECCS2PYTHON = "/home/marco/eccs2/python/bin/python3" # Input ECCS2INPUTDIR = "%s/input" % ECCS2DIR @@ -35,7 +35,7 @@ ECCS2STDERRIDP = "%s/stderr_idp_%s.log" % (ECCS2LOGSDIR,DAY) ECCS2FAILEDCMDIDP = "%s/failed-cmd-idp.sh" % ECCS2LOGSDIR # Number of processes to run in parallel -ECCS2NUMPROCESSES = 25 +ECCS2NUMPROCESSES = 10 # The 2 SPs that will be used to test each IdP ECCS2SPS = ["https://sp24-test.garr.it/Shibboleth.sso/Login?entityID=", "https://attribute-viewer.aai.switch.ch/Shibboleth.sso/Login?entityID="] diff --git a/runEccs2.py b/runEccs2.py index ed85b5fa0f467eab34cb2e0716c2ec50975e54ff..7b8917c73d4f38e11afd6a2c8fcbe33fb9391db9 100755 --- a/runEccs2.py +++ b/runEccs2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3 import argparse import asyncio diff --git a/utils.py b/utils.py index 03604e96c688a9b1c39fddb775812518d0495541..ea20f62538d36b5522e464663a685474e42a54bd 100644 --- a/utils.py +++ b/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +#!/usr/bin/env python3 import json import logging @@ -6,7 +6,7 @@ import pathlib import requests import sys -from eccs2properties import ECCS2SELENIUMLOGDIR, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT +from eccs2properties import ECCS2SELENIUMLOGDIR, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT, PATHCHROMEDRIVER from selenium import webdriver from selenium.common.exceptions import WebDriverException @@ -127,9 +127,9 @@ def getDriver(fqdn_idp=None,debugSelenium=False): # When debugging issues, it is helpful to enable more verbose logging.) try: if (debugSelenium and fqdn_idp): - driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)]) + driver = webdriver.Chrome(PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)]) else: - driver = webdriver.Chrome('chromedriver', options=chrome_options) + driver = webdriver.Chrome(PATHCHROMEDRIVER, options=chrome_options) except WebDriverException as e: sys.stderr.write("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!") sys.stderr.write(e.__str__())