diff --git a/.gitignore b/.gitignore index 79acb34829d834ce838f3ccb730f30e317387379..98a903a1eeba0314c5a0610407425a569270db49 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ ############################################################################# flowspy/settings.py +flowspy/settings.py.centos + *.pyc celerybeat-schedule *.log diff --git a/Dockerfile b/Dockerfile index 55a073cfc954bdc473e9785208a5f8fc2b17fe5b..4b80b2b28de46ed3b9c43f6050405b4f6f635eab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ RUN yum -y install procps RUN mkdir -p /var/log/fod /srv COPY . /srv/flowspy +#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf + RUN rm -rf /srv/flowspy/venv/ RUN (cd /srv/flowspy; ./install-centos.sh;) diff --git a/Dockerfiles.d/Dockerfile.centos.base b/Dockerfiles.d/Dockerfile.centos.base index 53878561c75c06b6545a8c160c276b805311a00a..4cf5cd502499a14c93bb2ee1286b55079be7b135 100644 --- a/Dockerfiles.d/Dockerfile.centos.base +++ b/Dockerfiles.d/Dockerfile.centos.base @@ -15,7 +15,8 @@ RUN yum -y install procps RUN mkdir -p /var/log/fod /srv #COPY . /srv/flowspy -COPY install-centos.sh requirements.txt requirements-centos.txt /srv/flowspy/ +COPY install-centos.sh install-centos*.sh requirements.txt* requirements-centos.txt /srv/flowspy/ +#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf RUN (cd /srv/flowspy; ./install-centos.sh --basesw;) diff --git a/Dockerfiles.d/Dockerfile.centos.step2 b/Dockerfiles.d/Dockerfile.centos.step2 index ae2e3b6a9ab82730b09b1202dbed6e0b1adce8fa..49377344c35f61b54d3992ca821c8a7d25de8542 100644 --- a/Dockerfiles.d/Dockerfile.centos.step2 +++ b/Dockerfiles.d/Dockerfile.centos.step2 @@ -18,6 +18,8 @@ ENV LC_ALL en_US.utf8 RUN mkdir -p /var/log/fod /srv COPY . /srv/flowspy +#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf + RUN rm -rf /srv/flowspy/venv/ RUN (cd /srv/flowspy; ./install-centos.sh --fodproper;) diff --git a/Dockerfiles.d/Dockerfile.centos.supervisord b/Dockerfiles.d/Dockerfile.centos.supervisord index b76e9fc6452a657c09963a515fa5207238546c1e..fb3441ff64d180193503e80478457253697c4bac 100644 --- a/Dockerfiles.d/Dockerfile.centos.supervisord +++ b/Dockerfiles.d/Dockerfile.centos.supervisord @@ -14,7 +14,7 @@ RUN yum -y install procps RUN mkdir -p /var/log/fod /srv #COPY . /srv/flowspy -COPY install-centos.sh requirements.txt requirements-centos.txt /srv/flowspy/ +COPY install-centos.sh install-centos*.sh requirements.txt* requirements-centos.txt /srv/flowspy/ RUN (cd /srv/flowspy; ./install-centos.sh --basesw;) diff --git a/Dockerfiles.d/Dockerfile.centos.supervisord.base b/Dockerfiles.d/Dockerfile.centos.supervisord.base index b281cc520992e25a128b8b87b99b582f582e9bff..a7c111d4deb6d0ad1b964145676fd0fb35c1eabb 100644 --- a/Dockerfiles.d/Dockerfile.centos.supervisord.base +++ b/Dockerfiles.d/Dockerfile.centos.supervisord.base @@ -18,7 +18,8 @@ RUN yum -y install procps RUN mkdir -p /var/log/fod /srv #COPY . /srv/flowspy -COPY install-centos.sh requirements.txt requirements-centos.txt /srv/flowspy/ +COPY install-centos.sh install-centos*.sh requirements.txt* requirements-centos.txt /srv/flowspy/ +#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf RUN (cd /srv/flowspy; ./install-centos.sh --basesw;) diff --git a/Dockerfiles.d/Dockerfile.debian.supervisord.base b/Dockerfiles.d/Dockerfile.debian.supervisord.base index 49d21abb1edc72e4481580714d4a00c093028f2c..5c173bb8c4f9c402ccac5ec69404e1894eaec525 100644 --- a/Dockerfiles.d/Dockerfile.debian.supervisord.base +++ b/Dockerfiles.d/Dockerfile.debian.supervisord.base @@ -20,7 +20,7 @@ RUN apt-get -qqy install locales RUN mkdir -p /var/log/fod /srv #COPY . /srv/flowspy -COPY install-debian.sh requirements.txt /srv/flowspy/ +COPY install-debian.sh requirements.txt* /srv/flowspy/ RUN apt-get -qqy install patch diff --git a/install-centos-fixcentos-sqlite.sh b/install-centos-fixcentos-sqlite.sh new file mode 100755 index 0000000000000000000000000000000000000000..c0fa3fcc5efb629976657eadd9a55544be9b5401 --- /dev/null +++ b/install-centos-fixcentos-sqlite.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +foddir="$1" +shift 1 + +#set -x + +if cat /etc/centos-release | grep -q "release 7."; then + + sqlite_version="$(sqlite3 -version)" + + if [ "$sqlite_version" != "${sqlite_version#3.7.}" ]; then + + echo "$0: sqlite_version=$sqlite_version is too old, trying to fix this" 1>&2 + + # https://stackoverflow.com/questions/60826836/improperlyconfiguredsqlite-3-8-3-or-later-is-required-found-s-database + + yum install -y make + cd ~ && curl https://www.sqlite.org/2020/sqlite-autoconf-3320100.tar.gz > sqlite-autoconf-3320100.tar.gz && tar xvfz sqlite-autoconf-3320100.tar.gz && cd sqlite-autoconf-3320100 && ./configure && make && make install + + ## + + echo 'export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"' >> "$foddir/fodenv.sh" + echo 'export LC_ALL=en_US.UTF-8' >> "$foddir/fodenv.sh" + echo 'export LANG=en_US.UTF-8' >> "$foddir/fodenv.sh" + + ## + +# if [ ! -f "$foddir/supervisord.conf" ]; then +# echo "$foddir/supervisord.conf not found, not adapting it" 1>&2 +# else +# yum install -y ed +# cp -f "$foddir/supervisord.conf" "$foddir/supervisord.conf.prep" +# #ed /etc/supervisord.conf <<EOF +# ed "$foddir/supervisord.conf.prep" <<EOF +#/^.program:gunicorn./ +#/^command=/ +#a +#environment=LD_LIBRARY_PATH=/usr/local/lib +#. +#/^.program:celeryd./ +#/^command=/ +#a +#environment=LD_LIBRARY_PATH=/usr/local/lib +#. +#w +#EOF +# +# fi + + fi + +fi + diff --git a/install-centos.sh b/install-centos.sh index de3fa40fa95761b5a568f465c2636557f658424f..87490dff858d4562557382e367aa16115f5362ed 100755 --- a/install-centos.sh +++ b/install-centos.sh @@ -1,26 +1,36 @@ +#!/bin/bash #!/bin/sh # # This script installs all dependencies for Firewall-on-Demand running in Python3 # with Celery, Redis, and sqlite. # +SCRIPT_NAME="install-centos.sh" + fod_dir="/srv/flowspy" venv_dir="/srv/venv" inside_docker=0 - install_basesw=1 install_fodproper=1 ensure_installed_pythonenv_wrapper=1 -# +# workaround for old Django with old OS sqlite3 (CENTOS7 only): +try_fixup_for_old_os_sqlite=1 +use_old_django_version__autodetect=1 -if [ -e "/.dockenv" ]; then +## +############################################################################## +############################################################################## + +if [ -e "/.dockerenv" ]; then echo "running inside docker assummed" 1>&2 inside_docker=1 fi +############################################################################## +############################################################################## while [ $# -gt 0 ]; do if [ $# -ge 1 -a "$1" = "--here" ]; then @@ -59,12 +69,21 @@ while [ $# -gt 0 ]; do done +if [ $# -gt 0 ]; then + echo "remaining unprocessed arguments: $*, aborting" 1>&2 + exit 2 +fi + +## + +#set -x + ## venv_dir_base="$(dirname "$venv_dir")" static_dir="$fod_dir/static" - + inst_dir="$(dirname "$0")" mkdir -p "$fod_dir" || exit @@ -78,10 +97,15 @@ fi echo "$0: inst_dir=$inst_dir fod_dir=$fod_dir => inst_dir_is_fod_dir=$inst_dir_is_fod_dir venv_dir=$venv_dir static_dir=$static_dir" 1>&2 #exit -## +############################################################################# +############################################################################# if [ "$install_basesw" = 1 ]; then + # requires ./install-centos-fixcentos-sqlite.sh in case of CENTOS7 + + echo "$0: step 1: installing base software dependencies (OS packages)" 1>&2 + set -e echo "Installing epel repo" @@ -99,126 +123,318 @@ if [ "$install_basesw" = 1 ]; then yum-config-manager --enable remi yum -q -y install redis + ## + + if [ "$try_fixup_for_old_os_sqlite" = 1 -a "$inside_docker" = 1 ]; then # only do try this fix inside docker env + + if ! type -p sqlite3; then + echo "$0: sqlite3 not found, installing it" 1>&2 + yum install -y sqlite3 + fi + + if [ "$try_fixup_for_old_os_sqlite" = 1 -a "$inside_docker" = 1 ]; then # only do try this fix inside docker env + . "./install-centos-fixcentos-sqlite.sh" "$fod_dir" + fi + + fi + set +e + echo "$0: step 1 done" 1>&2 + +fi + +## + +echo "$0: step 1a: handling sqlite3 too old fixup post actions" 1>&2 # only needed for CENTOS + +assume__sqlite_version__to_old=1 +if [ "$use_old_django_version__autodetect" = 1 ]; then + + sqlite_version="$(sqlite3 -version)" + if [ "$sqlite_version" != "${sqlite_version#3.7.}" ]; then + echo "$0: sqlite_version=$sqlite_version is too old, will use old Django version" 1>&2 + assume__sqlite_version__to_old=1 + else + echo "$0: sqlite_version=$sqlite_version is recent enough to use newer Django and dependencies" 1>&2 + assume__sqlite_version__to_old=0 + fi + fi ## +python_version="$(python3 --version | cut -d ' ' -f 2,2)" +if [ "$assume__sqlite_version__to_old" = 1 ]; then + echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2 + cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt" +elif [ -e "$fod_dir/requirements.txt.python$python_version" ]; then + echo "$0: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2 + cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt" +else + echo "$0: using $fod_dir/requirements.txt" 1>&2 +fi + +echo "$0: step 1b: preparing database system" 1>&2 + +############################################################################# +############################################################################# + if [ "$install_fodproper" = 0 ]; then + + echo "$0: step 2a: installing Python dependencies only" 1>&2 set -e echo "Setup partial python environment for FoD" - #virtualenv-3 /srv/venv - virtualenv-3 "$venv_dir" + if [ -x pyvenv ]; then + #pyvenv /srv/venv + pyvenv "$venv_dir" + else + #virtualenv-3 /srv/venv + virtualenv-3 "$venv_dir" + fi ln -sf "$venv_dir" "$fod_dir/venv" + #source /srv/venv/bin/activate source "$venv_dir/bin/activate" - #fix for broken anyjson and cl + ## + + # fix for broken anyjson and cl + # TODO: fix this more cleanly pip install 'setuptools<58' - cp -f requirements-centos.txt requirements.txt - pip install -r requirements.txt +# if [ "$assume__sqlite_version__to_old" = 1 ]; then +# +# # fix for broken anyjson and cl +# # TODO: fix this more cleanly +# #pip install 'setuptools<58' +# +# echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2 +# +# cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt" +# else +# +# python_version="$(python3 --version | cut -d ' ' -f 1,1)" +# if [ -e "$fod_dir/requirements.txt.$python_version" ]; then +# echo "$0: using python version specific $fod_dir/requirements.txt.$python_version" 1>&2 +# cp "$fod_dir/requirements.txt.$python_version" "$fod_dir/requirements.txt" +# fi +# +# fi + + (cd "$fod_dir" && pip install -r requirements.txt) + + echo "$0: step 2a done" 1>&2 else + echo "$0: step 2: installing FoD in installation dir + ensuring Python dependencies are installed + setting-up FoD settings, database preparations, and FoD run-time environment" 1>&2 + set -e + + echo "$0: step 2.0" 1>&2 + - echo "Setup python environment for FoD" mkdir -p /var/log/fod "$venv_dir_base" - #virtualenv-3 /srv/venv - virtualenv-3 "$venv_dir" + + ## + + echo "Setup python environment for FoD" + + if [ -x pyvenv ]; then + #pyvenv /srv/venv + pyvenv "$venv_dir" + else + #virtualenv-3 /srv/venv + virtualenv-3 "$venv_dir" + fi ln -sf "$venv_dir" "$fod_dir/venv" ( - set +e - #source /srv/venv/bin/activate - source "$venv_dir/bin/activate" - - #mkdir -p /srv/flowspy/log/ - mkdir -p "$fod_dir/log/" - - # Select source dir and copy FoD into /srv/flowspy/ - if [ "`basename "$0"`" = install-centos.sh ]; then - # this script is in the source directory - #cp -f -r "`dirname $0`"/* /srv/flowspy/ - cp -f -r "`dirname $0`"/* "$fod_dir" - elif [ -e /vagrant ]; then - # vagrant's copy in /vagrant/ - #cp -f -r /vagrant/* /srv/flowspy/ - cp -f -r /vagrant/* "$fod_dir" - elif [ -e ./install-centos.sh ]; then - # current directory is with the sourcecode - #cp -f -r ./* /srv/flowspy/ - cp -f -r ./* "$fod_dir" - else - echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./" - exit 1 - fi - - set -e - - #cd /srv/flowspy/ - cd "$fod_dir" - ( - cd flowspy - - if [ -e settings.py.centos ]; then - cp -f settings.py.centos settings.py - elif [ ! -e settings.py ]; then - #cp -f settings.py.dist settings.py - cp -f settings.py.centos.dist settings.py - patch settings.py < settings.py.patch - - sed -i "s#/srv/flowspy#$fod_dir#" "settings.py" - fi - ) - - cp -f requirements-centos.txt requirements.txt - - ## - - pip install 'setuptools<58' - pip install -r requirements.txt - - if [ ! -e "flowspy/settings_local.py" ]; then - touch flowspy/settings_local.py - fi - - #./manage.py syncdb --noinput - #mkdir -p /srv/flowspy/static/ - mkdir -p "$static_dir" - ./manage.py collectstatic --noinput - ./manage.py migrate - ./manage.py loaddata initial_data - - cp -f supervisord-centos.conf supervisord.conf - sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf" - - cp -f runfod.centos.sh runfod.sh - - mkdir -p /var/run/fod - #chown fod /var/run/fod - - ## - - if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then - echo "adding pythonev wrapper" 1>&2 - cat > "$fod_dir/pythonenv" <<EOF + set +e + #source /srv/venv/bin/activate + source "$venv_dir/bin/activate" + + ## + + #mkdir -p /srv/flowspy/log/ + mkdir -p "$fod_dir/log/" + + if true; then + + echo "$0: step 2.1: coyping from source dir to installation dir $fod_dir" 1>&2 + + MYSELF="$(basename "$0")" + DIRNAME="$(dirname "$0")" + + # Select source dir and copy FoD into /srv/flowspy/ + if [ "$MYSELF" = "$SCRIPT_NAME" ]; then # if started as main script, e.g., in Docker or on OS-installation + # this script is in the source directory + #cp -f -r "`dirname $0`"/* /srv/flowspy/ + #cp -f -r "$DIRNAME"/* "$fod_dir" + cp -f -r "$inst_dir"/* "$fod_dir" + elif [ -e /vagrant ]; then # running in vagrant with /vagrant available + # vagrant's copy in /vagrant/ + #cp -f -r /vagrant/* /srv/flowspy/ + cp -f -r /vagrant/* "$fod_dir" + elif [ -e "$SCRIPT_NAME" ]; then # running in vagrant with script current dir == install dir + # current directory is with the sourcecode + #cp -f -r ./* /srv/flowspy/ + cp -f -r ./* "$fod_dir" + else + echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./" + exit 1 + fi + + fi + + set -e + + echo "$0: step 2.2: setting-up FoD settings" 1>&2 + + #cd /srv/flowspy/ + cd "$fod_dir" + ( + cd flowspy # jump into settings subdir flowspy + + if [ "$inside_docker" = 1 -a -e settings.py.centos ]; then # user has own centos-specific settings prepared yet ? + + cp -f settings.py.centos.docker settings.py + + elif [ -e settings.py.centos ]; then # user has own centos-specific settings prepared yet ? + + cp -f settings.py.centos settings.py + + elif [ "$inside_docker" = 1 -a -e settings.py.docker ]; then # user has own settings prepared yet ? + + cp -f settings.py.docker settings.py + + elif [ -e settings.py ]; then # user has prepared a generic settings yet ? + + : # nothing todo + + else # prepare a settings.py from git repo's settings.py.dist/settings.py.centos.dist + settings.py.patch + + if [ "$assume__sqlite_version__to_old" = 1 ]; then + echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old using settings.py.centos.dist" 1>&2 + cp -f settings.py.centos.dist settings.py + else + cp -f settings.py.dist settings.py + fi + + patch settings.py < settings.py.patch + + sed -i "s#/srv/flowspy#$fod_dir#" "settings.py" + + fi + ) + + if [ ! -e "flowspy/settings_local.py" ]; then + touch flowspy/settings_local.py + fi + + echo "$0: step 2.3: ensuring Python dependencies are installed" 1>&2 + + if [ "$install_basesw" = 1 ]; then #are we running in --both mode, i.e. for the venv init is run for the first time, i.e. the problematic package having issues with to new setuptools is not yet installed? + # fix for broken anyjson and cl + # TODO: fix this more cleanly + pip install 'setuptools<58' + fi + + # actual proper installation of python requirements + pip install -r requirements.txt + + ## + + echo "$0: step 2.4: preparing FoD static files and database" 1>&2 + + echo "$0: step 2.4.1: preparing FoD static files" 1>&2 + + #mkdir -p /srv/flowspy/static/ + mkdir -p "$static_dir" + + ([ ! -f "fodenv.sh" ] || source "./fodenv.sh"; ./manage.py collectstatic --noinput) + + ## + + echo "$0: step 2.4.2.0: preparing DB and DB access" 1>&2 + + echo "$0: step 2.4.2.0: preparing FoD DB schema and basic data" 1>&2 + + echo "deploying/updating database schema" 1>&2 + ( + [ ! -f "fodenv.sh" ] || source "./fodenv.sh" + + #./manage.py syncdb --noinput + + #which sqlite3 + #sqlite3 -version + #source "$venv_dir/bin/activate" + + ./manage.py migrate + ./manage.py loaddata initial_data + ) + echo 1>&2 + + ## + + echo "$0: step 2.5: preparing FoD run-time environment" 1>&2 + + echo "$0: step 2.5.1: preparing necessary dirs" 1>&2 + + mkdir -p /var/run/fod + #chown fod /var/run/fod + + ## + + echo "$0: step 2.5.2: preparing FoD python wrapper" 1>&2 + + if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then + echo "adding pythonev wrapper" 1>&2 + cat > "$fod_dir/pythonenv" <<EOF #!/bin/bash . "$venv_dir/bin/activate" +[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh" exec "\$@" EOF - chmod +x "$fod_dir/pythonenv" - echo 1>&2 - fi + chmod +x "$fod_dir/pythonenv" + echo 1>&2 + fi + + ## + + echo "$0: step 2.5.3: preparing supervisord.conf" 1>&2 + + if [ "$assume__sqlite_version__to_old" = 1 -a -e "$fod_dir/supervisord-centos.conf" ]; then + echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using supervisord-centos.conf for old celery start syntax" 1>&2 + cp -f "$fod_dir/supervisord-centos.conf" "$fod_dir/supervisord.conf" + elif [ -e "$fod_dir/supervisord.conf.dist" ]; then + echo "providing supervisord config" 1>&2 + cp -f "$fod_dir/supervisord.conf.dist" "$fod_dir/supervisord.conf" + fi + + sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf" + echo 1>&2 + + ## + + echo "$0: step 2.5.4: preparing runfod script" 1>&2 + + if [ "$assume__sqlite_version__to_old" = 1 ]; then + echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using runfod.centos.sh for old celery start syntax" 1>&2 + cp -f runfod.centos.sh runfod.sh + fi - ## ) + + echo "$0: step 2 done" 1>&2 set +e fi +############################################################################# +############################################################################# + diff --git a/install-debian.sh b/install-debian.sh index 6916d2efca2e082fd4e9024c922f36c0d4e3c097..c5f6b0ca5fc79fdce36ced9d92a6534b60fee9ed 100755 --- a/install-debian.sh +++ b/install-debian.sh @@ -1,9 +1,12 @@ #!/bin/bash +#!/bin/sh # # This script installs all dependencies for Firewall-on-Demand running in Python3 # with Celery, Redis, and sqlite. # +SCRIPT_NAME="install-debian.sh" + fod_dir="/srv/flowspy" venv_dir="/srv/venv" @@ -94,10 +97,11 @@ function conf_db_access () { } +## ############################################################################## ############################################################################## -if [ -e "/.dockenv" ]; then +if [ -e "/.dockerenv" ]; then echo "running inside docker assummed" 1>&2 inside_docker=1 fi @@ -206,7 +210,7 @@ echo "conf_db_access=$conf_db_access DB_NAME=$DB__FOD_DBNAME DB_USER=$DB__FOD_US venv_dir_base="$(dirname "$venv_dir")" static_dir="$fod_dir/static" - + inst_dir="$(dirname "$0")" mkdir -p "$fod_dir" || exit @@ -220,8 +224,13 @@ fi echo "$0: inst_dir=$inst_dir fod_dir=$fod_dir => inst_dir_is_fod_dir=$inst_dir_is_fod_dir venv_dir=$venv_dir static_dir=$static_dir" 1>&2 #exit +############################################################################# +############################################################################# + if [ "$install_basesw" = 1 ]; then + echo "$0: step 1: installing base software dependencies (OS packages)" 1>&2 + if [ "$install_db" = "" ]; then mysql_server_pkg=("") elif [ "$install_db" = "mysql" ]; then @@ -292,10 +301,16 @@ if [ -n "$install_db" ]; then set +e + echo "$0: step 1 done" 1>&2 + fi ## +echo "$0: step 1a: handling sqlite3 too old fixup post actions" 1>&2 # only needed for CENTOS + +echo "$0: step 1b: preparing database system" 1>&2 + if [ -n "$init_db" ]; then set -e @@ -315,11 +330,27 @@ if [ -n "$init_db" ]; then set +e +## + +python_version="$(python3 --version | cut -d ' ' -f 2,2)" +#if [ "$assume__sqlite_version__to_old" = 1 ]; then +# echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2 +# cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt" +if [ -e "$fod_dir/requirements.txt.python$python_version" ]; then + echo "$0: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2 + cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt" +else + echo "$0: using $fod_dir/requirements.txt" 1>&2 fi -## +############################################################################# +############################################################################# if [ "$install_fodproper" = 0 ]; then + + echo "$0: step 2a: installing Python dependencies only" 1>&2 + + set -e echo "Setup partial python environment for FoD" @@ -337,19 +368,35 @@ if [ "$install_fodproper" = 0 ]; then #source /srv/venv/bin/activate source "$venv_dir/bin/activate" - # fix - pip install setuptools==57.5.0 + ## + # fix for broken anyjson and cl + # TODO: fix this more cleanly + pip install 'setuptools==57.5.0' pip install wheel + pip install -r requirements.txt -else + echo "$0: step 2a done" 1>&2 +else + + echo "$0: step 2: installing FoD in installation dir + ensuring Python dependencies are installed + setting-up FoD settings, database preparations, and FoD run-time environment" 1>&2 + + set -e + + echo "$0: step 2.0" 1>&2 + id fod &>/dev/null || useradd -m fod - echo "Setup python environment for FoD" + #mkdir -p /var/log/fod /srv mkdir -p /var/log/fod "$venv_dir_base" + + ## + + echo "Setup python environment for FoD" + if [ -x pyvenv ]; then #pyvenv /srv/venv pyvenv "$venv_dir" @@ -358,70 +405,99 @@ else virtualenv --python=python3 "$venv_dir" fi ln -sf "$venv_dir" "$fod_dir/venv" + ( - set +e - #source /srv/venv/bin/activate - source "$venv_dir/bin/activate" - #mkdir -p /srv/flowspy/ - mkdir -p "$fod_dir" - - if [ "$inst_dir_is_fod_dir" = 0 ]; then - - # Select source dir and copy FoD into /srv/flowspy/ - if [ "`basename "$0"`" = install-debian.sh ]; then - # this script is in the source directory - #cp -f -r "`dirname $0`"/* /srv/flowspy/ - cp -f -r "$inst_dir"/* "$fod_dir" - elif [ -e /vagrant ]; then - # vagrant's copy in /vagrant/ - #cp -f -r /vagrant/* /srv/flowspy/ - cp -f -r /vagrant/* "$fod_dir" - elif [ -e ./install-centos.sh ]; then - # current directory is with the sourcecode - #cp -f -r ./* /srv/flowspy/ - cp -f -r ./* "$fod_dir" - else - echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./" - exit 1 - fi + set +e + #source /srv/venv/bin/activate + source "$venv_dir/bin/activate" - fi + ## - #find "$fod_dir/" -not -user fod -exec chown -v fod: {} \; - find "$fod_dir/" -not -user fod -exec chown fod: {} \; - - set -e - - #cd /srv/flowspy/ - cd "$fod_dir" - ( - cd flowspy - - if [ ! -e settings.py ]; then - cp -f settings.py.dist settings.py - patch settings.py < settings.py.patch - - sed -i "s#/srv/flowspy#$fod_dir#" "settings.py" - fi - ) - - if [ "$install_basesw" = 1 ]; then #are we running in --both mode, i.e. for the venv init is run for the first time, i.e. the problematic package having issues with to new setuptools is not yet installed? - # fix - pip install setuptools==57.5.0 - : - fi + #mkdir -p /srv/flowspy/ + mkdir -p "$fod_dir" - pip install -r requirements.txt + if [ "$inst_dir_is_fod_dir" = 0 ]; then + + echo "$0: step 2.1: coyping from source dir to installation dir $fod_dir" 1>&2 + + MYSELF="$(basename "$0")" + + # Select source dir and copy FoD into /srv/flowspy/ + if [ "$MYSELF" = "$SCRIPT_NAME" ]; then # if started as main script, e.g., in Docker or on OS-installation + # this script is in the source directory + #cp -f -r "`dirname $0`"/* /srv/flowspy/ + cp -f -r "$inst_dir"/* "$fod_dir" + elif [ -e /vagrant ]; then # running in vagrant with /vagrant available + # vagrant's copy in /vagrant/ + #cp -f -r /vagrant/* /srv/flowspy/ + cp -f -r /vagrant/* "$fod_dir" + elif [ -e "$SCRIPT_NAME" ]; then # running in vagrant with script current dir == install dir + # current directory is with the sourcecode + #cp -f -r ./* /srv/flowspy/ + cp -f -r ./* "$fod_dir" + else + echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./" + exit 1 + fi - if [ ! -e "flowspy/settings_local.py" ]; then - touch flowspy/settings_local.py - fi + fi + + #find "$fod_dir/" -not -user fod -exec chown -v fod: {} \; + find "$fod_dir/" -not -user fod -exec chown fod: {} \; + + ### + + set -e + + echo "$0: step 2.2: setting-up FoD settings" 1>&2 + + #cd /srv/flowspy/ + cd "$fod_dir" + ( + cd flowspy # jump into settings subdir flowspy + + if [ "$inside_docker" = 1 -a -e settings.py.docker ]; then # user has own settings prepared yet ? + + cp -f settings.py.docker settings.py + + elif [ -e settings.py ]; then # user has prepared a generic settings yet ? + + : # nothing todo + + else # prepare from settings.py.dist + settings.py.patch + + cp -f settings.py.dist settings.py + patch settings.py < settings.py.patch + sed -i "s#/srv/flowspy#$fod_dir#" "settings.py" + + fi + ) + + if [ ! -e "flowspy/settings_local.py" ]; then + touch flowspy/settings_local.py + fi + + echo "$0: step 2.3: ensuring Python dependencies are installed" 1>&2 + + if [ "$install_basesw" = 1 ]; then #are we running in --both mode, i.e. for the venv init is run for the first time, i.e. the problematic package having issues with to new setuptools is not yet installed? + # fix for broken anyjson and cl + # TODO: fix this more cleanly + pip install 'setuptools==57.5.0' + fi + + # actual proper installation of python requirements + pip install -r requirements.txt + + ## + + echo "$0: step 2.4.1: preparing log sub dirs" 1>&2 mkdir -p "$fod_dir/log" "$fod_dir/logs" touch "$fod_dir/debug.log" chown -R fod: "$fod_dir/log" "$fod_dir/logs" "$fod_dir/debug.log" - + if [ "$try_install_docu" = 1 ]; then + echo "$0: step 2.4.2: compiling internal docu" 1>&2 echo "trying to install mkdocs-based documentation" 1>&2 ( set -e @@ -431,12 +507,19 @@ else ) fi - #./manage.py syncdb --noinput - #mkdir -p /srv/flowspy/static/ - mkdir -p "$static_dir" - ./manage.py collectstatic --noinput + ## + echo "$0: step 2.4: preparing FoD static files and database" 1>&2 + + echo "$0: step 2.4.1: preparing FoD static files" 1>&2 + + #mkdir -p /srv/flowspy/static/ + mkdir -p "$static_dir" + + ([ ! -f "fodenv.sh" ] || source "./fodenv.sh"; ./manage.py collectstatic --noinput) + + ## - ## + echo "$0: step 2.4.2.0: preparing DB and DB access" 1>&2 #if [ "$init_db" = "mariadb" -o "$init_db" = "mysql" ]; then # init_mysqllikedb "$DB__FOD_DBNAME" "$DB__FOD_USER" "$DB__FOD_PASSWORD" @@ -448,84 +531,104 @@ else echo 1>&2 fi - ## + ## - echo "deploying/updating database schema" 1>&2 - ./manage.py migrate - ./manage.py loaddata initial_data - echo 1>&2 - - # + echo "$0: step 2.4.2.0: preparing FoD DB schema and basic data" 1>&2 - # ./manage.py aboove may have created debug.log with root permissions: - chown -R fod: "$fod_dir/log" "$fod_dir/logs" "$fod_dir/debug.log" - [ ! -d "/var/log/fod" ] || chown -R fod: "/var/log/fod" + echo "deploying/updating database schema" 1>&2 + ( + [ ! -f "fodenv.sh" ] || source "./fodenv.sh" + + #./manage.py syncdb --noinput + + ./manage.py migrate + ./manage.py loaddata initial_data + ) + echo 1>&2 - # + ## - echo "providing supervisord config" 1>&2 - cp -f "$fod_dir/supervisord.conf.dist" "$fod_dir/supervisord.conf" - sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf" - echo 1>&2 + # ./manage.py above may have created debug.log with root permissions: + chown -R fod: "$fod_dir/log" "$fod_dir/logs" "$fod_dir/debug.log" + [ ! -d "/var/log/fod" ] || chown -R fod: "/var/log/fod" - # + # + echo "$0: step 2.5: preparing FoD run-time environment" 1>&2 - mkdir -p /var/run/fod - chown fod: /var/run/fod + echo "$0: step 2.5.1: preparing necessary dirs" 1>&2 - # + mkdir -p /var/run/fod + chown fod: /var/run/fod -## + ## - if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then - echo "adding pythonev wrapper" 1>&2 - cat > "$fod_dir/pythonenv" <<EOF + echo "$0: step 2.5.2: preparing FoD python wrapper" 1>&2 + + if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then + echo "adding pythonev wrapper" 1>&2 + cat > "$fod_dir/pythonenv" <<EOF #!/bin/bash . "$venv_dir/bin/activate" +[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh" exec "\$@" EOF - chmod +x "$fod_dir/pythonenv" - echo 1>&2 - fi + chmod +x "$fod_dir/pythonenv" + echo 1>&2 + fi -## + ## - fod_systemd_dir="$fod_dir/systemd" - cp -f "$fod_systemd_dir/fod-gunicorn.service.dist" "$fod_systemd_dir/fod-gunicorn.service" - sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-gunicorn.service" + echo "$0: step 2.5.3: preparing supervisord.conf" 1>&2 - cp -f "$fod_systemd_dir/fod-celeryd.service.dist" "$fod_systemd_dir/fod-celeryd.service" - sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-celeryd.service" + cp -f "$fod_dir/supervisord.conf.dist" "$fod_dir/supervisord.conf" + sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf" + echo 1>&2 - cp -f "$fod_systemd_dir/fod-status-email-user@.service.dist" "$fod_systemd_dir/fod-status-email-user@.service" - sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-status-email-user@.service" + ## - if [ "$install_systemd_services" = 1 ]; then - echo 1>&2 - echo "Installing systemd services" 1>&2 - echo 1>&2 - #cp -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "/etc/systemd/system/" - cp -v -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "$fod_systemd_dir/fod-status-email-user@.service" "/etc/systemd/system/" 1>&2 - systemctl daemon-reload + + echo "$0: step 2.5.5: preparing systemd files" 1>&2 - systemctl enable fod-gunicorn - systemctl enable fod-celeryd + fod_systemd_dir="$fod_dir/systemd" + cp -f "$fod_systemd_dir/fod-gunicorn.service.dist" "$fod_systemd_dir/fod-gunicorn.service" + sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-gunicorn.service" - systemctl restart fod-gunicorn - systemctl restart fod-celeryd + cp -f "$fod_systemd_dir/fod-celeryd.service.dist" "$fod_systemd_dir/fod-celeryd.service" + sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-celeryd.service" - sleep 5 - SYSTEMD_COLORS=1 systemctl status fod-gunicorn | cat - echo - SYSTEMD_COLORS=1 systemctl status fod-celeryd | cat - echo + cp -f "$fod_systemd_dir/fod-status-email-user@.service.dist" "$fod_systemd_dir/fod-status-email-user@.service" + sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-status-email-user@.service" - fi + if [ "$install_systemd_services" = 1 ]; then + echo 1>&2 + echo "Installing systemd services" 1>&2 + echo 1>&2 + #cp -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "/etc/systemd/system/" + cp -v -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "$fod_systemd_dir/fod-status-email-user@.service" "/etc/systemd/system/" 1>&2 + systemctl daemon-reload + + systemctl enable fod-gunicorn + systemctl enable fod-celeryd + + systemctl restart fod-gunicorn + systemctl restart fod-celeryd + + sleep 5 + SYSTEMD_COLORS=1 systemctl status fod-gunicorn | cat + echo + SYSTEMD_COLORS=1 systemctl status fod-celeryd | cat + echo + + fi ) + + echo "$0: step 2 done" 1>&2 set +e fi +############################################################################# +############################################################################# diff --git a/requirements.txt.python3.6.8 b/requirements.txt.python3.6.8 new file mode 100644 index 0000000000000000000000000000000000000000..77ddc870c63234df08e9ed91cfc49b7bbc2245df --- /dev/null +++ b/requirements.txt.python3.6.8 @@ -0,0 +1,42 @@ +setuptools==59.1.1 +argparse +cl +celery==5.2.0b3 +django-admin +django-picklefield==2.0 +django-registration==3.1.2 +djangorestframework==3.12.0 +django-tinymce +django-form-utils +django-settings-export +Django==2.2.28 +gevent +greenlet +gunicorn +kombu==5.2.0rc1 +lxml +mailer +mysqlclient +ncclient +paramiko +pycrypto +pyparsing +python-dateutil +python-memcached +PyYAML +#wsgiref +six +requests +pysnmp +ply +ipaddress +idna +appdirs +anyjson +cryptography +redis +pytest==6.2.5 +pytest-django==4.4.0 +psutil +longerusername +intervaltree diff --git a/runfod-supervisord.sh b/runfod-supervisord.sh index e21c6901449a94a71dcb4f4fd42ce615c7972518..a3ed66379ff96073f20e1bba411621fce88d2693 100755 --- a/runfod-supervisord.sh +++ b/runfod-supervisord.sh @@ -40,7 +40,12 @@ while [ $# -gt 0 ]; do done -## +############################################################################## +############################################################################## + +if [ -e "$fod_dir/fodenv.sh" ]; then + . "$fod_dir/fodenv.sh" +fi #. /srv/venv/bin/activate . "$venv_dir/bin/activate" @@ -51,6 +56,7 @@ if [ ! -e "$fod_dir/pythonenv" ]; then cat > "$fod_dir/pythonenv" <<EOF #!/bin/bash . "$venv_dir/bin/activate" +[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh" exec "\$@" EOF #chmod +x /srv/flowspy/pythonenv @@ -102,7 +108,7 @@ systemctl disable redis #useradd -m fod ( - set -x + #set -x #source /srv/venv/bin/activate source "$venv_dir/bin/activate" #cd /srv/flowspy && ./manage.py collectstatic --noinput @@ -113,7 +119,13 @@ systemctl disable redis sysctl vm.overcommit_memory=1 # supervisord.conf -cp -f supervisord.conf /etc +if [ -f supervisord.conf.prep ]; then + echo "$0: using supervisord.conf.prep" 1>&2 + cp -f supervisord.conf.prep /etc/supervisord.conf +else + echo "$0: using supervisord.conf" 1>&2 + cp -f supervisord.conf /etc +fi mkdir -p /var/run/fod /var/log/fod @@ -128,3 +140,6 @@ chown fod "$fod_dir" "$fod_dir/log" "$fod_dir/logs" "$fod_dir/log/"* "$fod_dir/l mkdir -p /var/run/supervisor exec /usr/bin/supervisord -n -c /etc/supervisord.conf +############################################################################## +############################################################################## + diff --git a/runfod.centos.sh b/runfod.centos.sh index 9e258a4f3c28c698b83db3d43fc9161a96dde29b..ef3f28d33f0eb738708f58cf9ed103766432584c 100755 --- a/runfod.centos.sh +++ b/runfod.centos.sh @@ -47,17 +47,27 @@ while [ $# -gt 0 ]; do done +############################################################################## +############################################################################## + +if [ -e "$fod_dir/fodenv.sh" ]; then + . "$fod_dir/fodenv.sh" +fi + ## #. /srv/venv/bin/activate . "$venv_dir/bin/activate" +## + #if [ ! -e /srv/flowspy/pythonenv ]; then if [ ! -e "$fod_dir/pythonenv" ]; then #cat > /srv/flowspy/pythonenv <<EOF cat > "$fod_dir/pythonenv" <<EOF #!/bin/bash . "$venv_dir/bin/activate" +[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh" exec "\$@" EOF #chmod +x /srv/flowspy/pythonenv @@ -93,4 +103,6 @@ echo "Starting FoD gunicorn in foreground" 1>&2 #exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi -w 1 -k gevent --limit-request-fields 10000 --timeout 30 #--preload exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi --reload -w 1 -k gevent --limit-request-fields 10000 --timeout 30 +############################################################################## +############################################################################## diff --git a/runfod.sh b/runfod.sh index be6f964e631795709856138bba38e20b2d9da8c1..60be3c0f158953833b1670f42bba6b6a41f9967a 100755 --- a/runfod.sh +++ b/runfod.sh @@ -47,17 +47,27 @@ while [ $# -gt 0 ]; do done +############################################################################## +############################################################################## + +if [ -e "$fod_dir/fodenv.sh" ]; then + . "$fod_dir/fodenv.sh" +fi + ## #. /srv/venv/bin/activate . "$venv_dir/bin/activate" +## + #if [ ! -e /srv/flowspy/pythonenv ]; then if [ ! -e "$fod_dir/pythonenv" ]; then #cat > /srv/flowspy/pythonenv <<EOF cat > "$fod_dir/pythonenv" <<EOF #!/bin/bash . "$venv_dir/bin/activate" +[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh" exec "\$@" EOF #chmod +x /srv/flowspy/pythonenv @@ -93,4 +103,6 @@ echo "Starting FoD gunicorn in foreground" 1>&2 #exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi -w 1 -k gevent --limit-request-fields 10000 --timeout 30 #--preload exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi --reload -w 1 -k gevent --limit-request-fields 10000 --timeout 30 +############################################################################## +##############################################################################