From 33b77769e148ff12332ae2b616f50b89cfabac59 Mon Sep 17 00:00:00 2001 From: Marco Malavolti <marco.malavolti@garr.it> Date: Fri, 28 Jul 2023 18:00:14 +0200 Subject: [PATCH] Docker implementation update --- Dockerfile | 56 ------------------------------------ Dockerfile-dev | 69 +++++++++++++++++++++++++++++++++++++++++++++ README-Docker.rst | 38 +++++++++++++++++++++++++ cleanAndRunEccs.sh | 2 +- docker-compose.yml | 14 +++++++++ dot-env-template | 4 +++ eccs-sps-md-cron | 1 + get-sps-metadata.sh | 42 +++++++++++++++++++++++++++ setup-eccs-dev.sh | 49 ++++++++++++++++++++++++++++++++ start.sh | 6 ++++ supervisord.conf | 17 +++++++++++ 11 files changed, 241 insertions(+), 57 deletions(-) delete mode 100644 Dockerfile create mode 100644 Dockerfile-dev create mode 100644 docker-compose.yml create mode 100644 dot-env-template create mode 100644 eccs-sps-md-cron create mode 100755 get-sps-metadata.sh create mode 100755 setup-eccs-dev.sh create mode 100644 start.sh create mode 100644 supervisord.conf diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e21216b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM debian:12 -LABEL Authors="Marco Malavolti <marco.malavolti@garr.it>" - -USER root -ENV DEBIAN_FRONTEND=noninteractive -ENV ECCS_VERSION=2.1.0 -ENV XMLSECTOOL_VERSION=3.0.0 -ENV JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto - -COPY --from=hairyhenderson/gomplate:v3.11.5 /gomplate /bin/gomplate - -RUN apt-get update \ - && apt-get install --no-install-recommends -y apt-utils vim git bash-completion ca-certificates curl unzip uwsgi cron gpg gpg-agent \ - python3 python3-pip python3-click python3-flask python3-flask-restful python3-requests python3-selenium python3-urllib3 \ - #&& cp /etc/apt/sources.list /etc/apt/sources.list.d/sources-src.list \ - && sed -i -e 's/"syntax on/syntax on/g' /etc/vim/vimrc \ - && printf "\nif [ -f /etc/bash_completion ]; then\n . /etc/bash_completion\nfi" >> /etc/profile - -# Get ECCS -WORKDIR /root -RUN curl "https://gitlab.software.geant.org/edugain/eccs/-/archive/v$ECCS_VERSION/eccs-v$ECCS_VERSION.tar.gz" --output eccs-v$ECCS_VERSION.tar.gz -RUN tar xzf eccs-v$ECCS_VERSION.tar.gz && rm eccs-v$ECCS_VERSION.tar.gz -RUN mv eccs-v$ECCS_VERSION eccs - -# Get Google Chrome & Google Chrome Driver -WORKDIR eccs -RUN curl "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" --output google-chrome-stable_current_amd64.deb -RUN apt install -y ./google-chrome-stable_current_amd64.deb --no-install-recommends -RUN curl "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/115.0.5790.110/linux64/chromedriver-linux64.zip" --output chromedriver_linux64.zip -RUN unzip chromedriver_linux64.zip -RUN rm chromedriver_linux64.zip google-chrome-stable_current_amd64.deb -RUN cd /root - -# Get XMLSecTool -RUN wget "https://corretto.aws/downloads/resources/11.0.6.10.1/B04F24E3.pub" -O /tmp/amazon-corretto.pub \ - && gpg --no-default-keyring --keyring /tmp/temp-keyring.gpg --import /tmp/amazon-corretto.pub \ - && gpg --no-default-keyring --keyring /tmp/temp-keyring.gpg --export --output /etc/apt/keyrings/amazon-corretto.gpg \ - && rm /tmp/temp-keyring.gpg - -COPY amazon-corretto.list /etc/apt/sources.list.d/amazon-corretto.list -RUN apt-get update && apt-get install -y java-11-amazon-corretto-jdk - -RUN curl "https://shibboleth.net/downloads/tools/xmlsectool/$XMLSECTOOL_VERSION/xmlsectool-$XMLSECTOOL_VERSION-bin.zip" --output xmlsectool-$XMLSECTOOL_VERSION-bin.zip -RUN unzip xmlsectool-$XMLSECTOOL_VERSION-bin.zip - -COPY eccs_properties.py.template eccs_properties.py - -EXPOSE 80 -EXPOSE 443 - -CMD tail -f /dev/null - -##COPY start-dev.sh /start.sh -#COPY start.sh /start.sh -#RUN chmod +x /start.sh -#CMD /start.sh diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 0000000..a712a79 --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,69 @@ +FROM debian:12 +LABEL Authors="Marco Malavolti <marco.malavolti@garr.it>" + +USER root + +ENV ECCS_VERSION=2.1.0 +ENV XMLSECTOOL_VERSION=3.0.0 +ENV AMAZON_JDK_KEY=https://corretto.aws/downloads/resources/11.0.6.10.1/B04F24E3.pub + +COPY --from=hairyhenderson/gomplate:v3.11.5 /gomplate /bin/gomplate + +RUN apt-get update \ + && apt-get install --no-install-recommends -y apt-utils vim git bash-completion \ + ca-certificates curl unzip uwsgi cron gpg gpg-agent libxml2-utils supervisor \ + python3 python3-pip python3-click python3-flask python3-flask-restful \ + python3-requests python3-selenium python3-urllib3 apache2 \ + libpcre3 libpcre3-dev libapache2-mod-proxy-uwsgi build-essential python3-dev \ + && sed -i -e 's/"syntax on/syntax on/g' /etc/vim/vimrc \ + && printf "\nif [ -f /etc/bash_completion ]; then\n . /etc/bash_completion\nfi" >> /etc/profile + +COPY supervisord.conf /etc/supervisor/conf.d/ + +# Get ECCS +WORKDIR /root +RUN curl "https://gitlab.software.geant.org/edugain/eccs/-/archive/v$ECCS_VERSION/eccs-v$ECCS_VERSION.tar.gz" --output eccs-v$ECCS_VERSION.tar.gz \ + && tar xzf eccs-v$ECCS_VERSION.tar.gz && rm eccs-v$ECCS_VERSION.tar.gz \ + && mv eccs-v$ECCS_VERSION eccs + +# Get Google Chrome & Google Chrome Driver +WORKDIR eccs +RUN curl "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" --output google-chrome-stable_current_amd64.deb \ + && apt install -y ./google-chrome-stable_current_amd64.deb --no-install-recommends \ + && curl "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/115.0.5790.110/linux64/chromedriver-linux64.zip" --output chromedriver_linux64.zip \ + && unzip chromedriver_linux64.zip \ + && rm chromedriver_linux64.zip google-chrome-stable_current_amd64.deb \ + && mv chromedriver-linux64/chromedriver . \ + && rm -rf chromedriver-linux64 + +# Get XMLSecTool & Amazon Corretto 11 JDK +RUN wget $AMAZON_JDK_KEY -O /tmp/amazon-corretto.pub \ + && gpg --no-default-keyring --keyring /tmp/temp-keyring.gpg --import /tmp/amazon-corretto.pub \ + && gpg --no-default-keyring --keyring /tmp/temp-keyring.gpg --export --output /etc/apt/keyrings/amazon-corretto.gpg \ + && rm /tmp/temp-keyring.gpg +COPY amazon-corretto.list /etc/apt/sources.list.d/amazon-corretto.list +RUN apt-get update && apt-get install -y java-11-amazon-corretto-jdk \ + && curl "https://shibboleth.net/downloads/tools/xmlsectool/$XMLSECTOOL_VERSION/xmlsectool-$XMLSECTOOL_VERSION-bin.zip" --output xmlsectool-$XMLSECTOOL_VERSION-bin.zip \ + && unzip xmlsectool-$XMLSECTOOL_VERSION-bin.zip \ + && rm xmlsectool-$XMLSECTOOL_VERSION-bin.zip \ + && curl "https://mdx.idem.garr.it/idem-mdx-service-crt.pem" --output idem-mdx-service-crt.pem + +COPY get-sps-metadata.sh get-sps-metadata.sh +COPY eccs_properties.py.template eccs_properties.py +COPY eccs-sps-md-cron /etc/cron.d/eccs_get_sps_metadata + +# Install ECCS API +COPY eccs.ini eccs.ini +COPY eccs.service eccs.service +COPY eccs.service /etc/systemd/system/eccs.service + + +EXPOSE 80 +EXPOSE 443 + +CMD tail -f /dev/null + +##COPY start-dev.sh /start.sh +#COPY start.sh /start.sh +#RUN chmod +x /start.sh +#CMD /start.sh diff --git a/README-Docker.rst b/README-Docker.rst index 4d45bfc..6627dcc 100644 --- a/README-Docker.rst +++ b/README-Docker.rst @@ -26,6 +26,43 @@ Setup dev environment * ``git clone git@gitlab.software.geant.org:edugain/eccs.git $HOME/eccs`` +#. Create the `.env` file: + + * ``cd $HOME/eccs`` + + * ``cp dot-env-template .env`` + +Start dev environment +--------------------- + +* ``cd $HOME/eccs`` + +* ``bash setup-eccs-dev.sh`` + + +Open a terminal on the container docker +--------------------------------------- + +``docker exec -it -u root eccs bash`` + +[`Indice`_] + +Container Registry Authentication +--------------------------------- + +``docker login gitlab.software.geant.org:5050`` + +Build a new docker container image +---------------------------------- + +``docker build -t gitlab.software.geant.org:5050/edugain/eccs -f Dockerfile .`` + +Load a container image to the Container Registry +------------------------------------------------ + +``docker push gitlab.software.geant.org:5050/edugain/eccs:MAJOR.MINOR.PATCH`` + +We'll respect `Semantic Versioning`_. Authors ------- @@ -35,5 +72,6 @@ Authors .. _post-installation: https://docs.docker.com/engine/install/linux-postinstall/ .. _Docker: https://docs.docker.com/engine/install/ +.. _Semantic Versioning: https://semver.org .. _Marco Malavolti: mailto:marco.malavolti@garr.it .. _Valentin Pocotilenco: mailto:valentin.pocotilenco@renam.md diff --git a/cleanAndRunEccs.sh b/cleanAndRunEccs.sh index cfb7fa1..bc5af67 100755 --- a/cleanAndRunEccs.sh +++ b/cleanAndRunEccs.sh @@ -4,7 +4,7 @@ BASEDIR=$HOME -source $HOME/.bash_profile +#source $HOME/.bash_profile # Remove old IdP and Fed List rm -f $BASEDIR/eccs/input/*.json diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8b2dd09 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.9' +services: + eccs: + image: gitlab.software.geant.org:5050/edugain/eccs:dev + build: + context: . + dockerfile: Dockerfile-dev + container_name: eccs + hostname: eccs + env_file: + - .env + ports: + - 80:80 + - 443:443 diff --git a/dot-env-template b/dot-env-template new file mode 100644 index 0000000..adbb711 --- /dev/null +++ b/dot-env-template @@ -0,0 +1,4 @@ +DEBIAN_FRONTEND=noninteractive +ECCS_VERSION=2.1.0 +XMLSECTOOL_VERSION=3.0.0 +JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto diff --git a/eccs-sps-md-cron b/eccs-sps-md-cron new file mode 100644 index 0000000..749b217 --- /dev/null +++ b/eccs-sps-md-cron @@ -0,0 +1 @@ +0 3 * * * /bin/bash $HOME/eccs/get-sps-metadata.sh diff --git a/get-sps-metadata.sh b/get-sps-metadata.sh new file mode 100755 index 0000000..4aa5422 --- /dev/null +++ b/get-sps-metadata.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +sp_md_1="input/sp_md_1.xml" +sp_md_2="input/sp_md_2.xml" +output_file="input/sps-metadata.xml" + +# Download SPs metadata only if they are changes since last download +curl -z $sp_md_1 -o $sp_md_1 "https://mdx.idem.garr.it/edugain/entities/https:%2F%2Fattribute-viewer.aai.switch.ch%2Finterfederation-test%2Fshibboleth" +curl -z $sp_md_2 -o $sp_md_2 "https://mdx.idem.garr.it/edugain/entities/https:%2F%2Fsp-demo.idem.garr.it%2Fshibboleth" + +# Check the existance of the metadata files +if [ ! -f "$sp_md_1" ] || [ ! -f "$sp_md_2" ]; then + echo "Error: both files have to exist." + exit 1 +fi + +sp_md_1_is_valid=$(bash xmlsectool-3.0.0/xmlsectool.sh --verifySignature --certificate idem-mdx-service-crt.pem --inFile input/sp_md_1.xml | grep "XML document signature verified." | wc -l) +sp_md_2_is_valid=$(bash xmlsectool-3.0.0/xmlsectool.sh --verifySignature --certificate idem-mdx-service-crt.pem --inFile input/sp_md_2.xml | grep "XML document signature verified." | wc -l) + +# Check the validity of both SP metadata files +if [ $sp_md_1_is_valid -eq 0 ] || [ $sp_md_1_is_valid -eq 0 ]; then + echo "Error: at least one of SP metadata file has an invalid signature." + exit 1 +fi + +# Remove XML declaration from both SP Metadata files +sed -i '1d' "$sp_md_1" +sed -i '1d' "$sp_md_2" + + +header='<?xml version="1.0" encoding="UTF-8"?> +<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata">' + +footer='</md:EntitiesDescriptor>' + +# Generation the sps-metadata.xml +echo "$header" > "$output_file" +cat "$sp_md_1" >> "$output_file" +cat "$sp_md_2" >> "$output_file" +echo "$footer" >> "$output_file" + +xmllint --format "$output_file" > "$output_file.tmp" && mv "$output_file.tmp" "$output_file" diff --git a/setup-eccs-dev.sh b/setup-eccs-dev.sh new file mode 100755 index 0000000..dc9f153 --- /dev/null +++ b/setup-eccs-dev.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +function modify_hosts_file() { + local action="$1" + local domain_name="$2" + local ip_address="$3" + local hosts_path="/etc/hosts" + + if [[ "$action" == "add" ]]; then + if sudo grep -q "$ip_address $domain_name" "$hosts_path"; then + return 0 + fi + echo -ne "\n$ip_address\t$domain_name" | sudo tee -a "$hosts_path" >/dev/null + #echo "\nAdded $ip_address $domain_name to /etc/hosts file" + elif [[ "$action" == "remove" ]]; then + if ! sudo grep -q "$domain_name" "$hosts_path"; then + return 0 + fi + sudo sed -i "/$domain_name/d" "$hosts_path" + #echo "\nRemoved line containing $domain_name from /etc/hosts file" + else + echo "Utilizzo: modify_hosts_file [add|remove] <dominio> [<indirizzo IP>]" + return 1 + fi +} + +echo "Stop and remove all" +docker compose down + +#echo "Stop any local Apache2 Web Server started" +#sudo systemctl stop apache2.service + +echo "Remove old container image to be able to create it from scratch" +docker rmi gitlab.software.geant.org:5050/edugain/eccs:dev + +echo "Start docker container creation" +docker compose up -d + +ECCS_IP=$(docker inspect eccs | grep "IPAddress" | grep -E -o '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | tail -n 1) + +DOMAIN_NAME="technical.edugain.org technical-test.edugain.org" + +modify_hosts_file "remove" "$DOMAIN_NAME" +modify_hosts_file "add" "$DOMAIN_NAME" "$ECCS_IP" + +echo "" +echo "Per accedere al container 'eccs' usare:" +echo "" +echo "docker exec -it -u root eccs bash" diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..995b731 --- /dev/null +++ b/start.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# ...other things... + +# Last command +exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/supervisord.conf b/supervisord.conf new file mode 100644 index 0000000..b7ff44a --- /dev/null +++ b/supervisord.conf @@ -0,0 +1,17 @@ +[supervisord] +nodaemon=true + +[program:cron] +command=cron -f +autostart=true +autorestart=true + +#[program:uwsgi] +#command=uwsgi --ini /path/to/uwsgi_config.ini +#autostart=true +#autorestart=true + +#[program:apache2] +#command=apache2ctl -DFOREGROUND +#autostart=true +#autorestart=true -- GitLab