Skip to content
Snippets Groups Projects
Commit 7afcfe2f authored by David Schmitz's avatar David Schmitz
Browse files

Dockerfiles, runfod, install: fix CENTOS7 issues with too old sqlite3

parent 65dc75a1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 677 additions and 215 deletions
.gitignore
+ 2
− 0
View file @ 7afcfe2f
......@@ -3,6 +3,8 @@
#############################################################################
flowspy/settings.py
flowspy/settings.py.centos
*.pyc
celerybeat-schedule
*.log
......
Dockerfile
+ 2
− 0
View file @ 7afcfe2f
......@@ -7,6 +7,8 @@ RUN yum -y install procps
RUN mkdir -p /var/log/fod /srv
COPY . /srv/flowspy
#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf
RUN rm -rf /srv/flowspy/venv/
RUN (cd /srv/flowspy; ./install-centos.sh;)
......
Dockerfiles.d/Dockerfile.centos.base
+ 2
− 1
View file @ 7afcfe2f
......@@ -15,7 +15,8 @@ RUN yum -y install procps
RUN mkdir -p /var/log/fod /srv
#COPY . /srv/flowspy
COPY install-centos.sh requirements.txt requirements-centos.txt /srv/flowspy/
COPY install-centos.sh install-centos*.sh requirements.txt* requirements-centos.txt /srv/flowspy/
#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf
RUN (cd /srv/flowspy; ./install-centos.sh --basesw;)
......
Dockerfiles.d/Dockerfile.centos.step2
+ 2
− 0
View file @ 7afcfe2f
......@@ -18,6 +18,8 @@ ENV LC_ALL en_US.utf8
RUN mkdir -p /var/log/fod /srv
COPY . /srv/flowspy
#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf
RUN rm -rf /srv/flowspy/venv/
RUN (cd /srv/flowspy; ./install-centos.sh --fodproper;)
......
Dockerfiles.d/Dockerfile.centos.supervisord
+ 1
− 1
View file @ 7afcfe2f
......@@ -14,7 +14,7 @@ RUN yum -y install procps
RUN mkdir -p /var/log/fod /srv
#COPY . /srv/flowspy
COPY install-centos.sh requirements.txt requirements-centos.txt /srv/flowspy/
COPY install-centos.sh install-centos*.sh requirements.txt* requirements-centos.txt /srv/flowspy/
RUN (cd /srv/flowspy; ./install-centos.sh --basesw;)
......
Dockerfiles.d/Dockerfile.centos.supervisord.base
+ 2
− 1
View file @ 7afcfe2f
......@@ -18,7 +18,8 @@ RUN yum -y install procps
RUN mkdir -p /var/log/fod /srv
#COPY . /srv/flowspy
COPY install-centos.sh requirements.txt requirements-centos.txt /srv/flowspy/
COPY install-centos.sh install-centos*.sh requirements.txt* requirements-centos.txt /srv/flowspy/
#COPY supervisord-centos.conf /srv/flowspy/supervisord.conf
RUN (cd /srv/flowspy; ./install-centos.sh --basesw;)
......
Dockerfiles.d/Dockerfile.debian.supervisord.base
+ 1
− 1
View file @ 7afcfe2f
......@@ -20,7 +20,7 @@ RUN apt-get -qqy install locales
RUN mkdir -p /var/log/fod /srv
#COPY . /srv/flowspy
COPY install-debian.sh requirements.txt /srv/flowspy/
COPY install-debian.sh requirements.txt* /srv/flowspy/
RUN apt-get -qqy install patch
......
install-centos-fixcentos-sqlite.sh 0 → 100755
+ 54
− 0
View file @ 7afcfe2f
#!/bin/bash
foddir="$1"
shift 1
#set -x
if cat /etc/centos-release | grep -q "release 7."; then
sqlite_version="$(sqlite3 -version)"
if [ "$sqlite_version" != "${sqlite_version#3.7.}" ]; then
echo "$0: sqlite_version=$sqlite_version is too old, trying to fix this" 1>&2
# https://stackoverflow.com/questions/60826836/improperlyconfiguredsqlite-3-8-3-or-later-is-required-found-s-database
yum install -y make
cd ~ && curl https://www.sqlite.org/2020/sqlite-autoconf-3320100.tar.gz > sqlite-autoconf-3320100.tar.gz && tar xvfz sqlite-autoconf-3320100.tar.gz && cd sqlite-autoconf-3320100 && ./configure && make && make install
##
echo 'export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"' >> "$foddir/fodenv.sh"
echo 'export LC_ALL=en_US.UTF-8' >> "$foddir/fodenv.sh"
echo 'export LANG=en_US.UTF-8' >> "$foddir/fodenv.sh"
##
# if [ ! -f "$foddir/supervisord.conf" ]; then
# echo "$foddir/supervisord.conf not found, not adapting it" 1>&2
# else
# yum install -y ed
# cp -f "$foddir/supervisord.conf" "$foddir/supervisord.conf.prep"
# #ed /etc/supervisord.conf <<EOF
# ed "$foddir/supervisord.conf.prep" <<EOF
#/^.program:gunicorn./
#/^command=/
#a
#environment=LD_LIBRARY_PATH=/usr/local/lib
#.
#/^.program:celeryd./
#/^command=/
#a
#environment=LD_LIBRARY_PATH=/usr/local/lib
#.
#w
#EOF
#
# fi
fi
fi
install-centos.sh
+ 307
− 91
View file @ 7afcfe2f
#!/bin/bash
#!/bin/sh
#
# This script installs all dependencies for Firewall-on-Demand running in Python3
# with Celery, Redis, and sqlite.
#
SCRIPT_NAME="install-centos.sh"
fod_dir="/srv/flowspy"
venv_dir="/srv/venv"
inside_docker=0
install_basesw=1
install_fodproper=1
ensure_installed_pythonenv_wrapper=1
#
# workaround for old Django with old OS sqlite3 (CENTOS7 only):
try_fixup_for_old_os_sqlite=1
use_old_django_version__autodetect=1
if [ -e "/.dockenv" ]; then
##
##############################################################################
##############################################################################
if [ -e "/.dockerenv" ]; then
echo "running inside docker assummed" 1>&2
inside_docker=1
fi
##############################################################################
##############################################################################
while [ $# -gt 0 ]; do
if [ $# -ge 1 -a "$1" = "--here" ]; then
......@@ -59,12 +69,21 @@ while [ $# -gt 0 ]; do
done
if [ $# -gt 0 ]; then
echo "remaining unprocessed arguments: $*, aborting" 1>&2
exit 2
fi
##
#set -x
##
venv_dir_base="$(dirname "$venv_dir")"
static_dir="$fod_dir/static"
inst_dir="$(dirname "$0")"
mkdir -p "$fod_dir" || exit
......@@ -78,10 +97,15 @@ fi
echo "$0: inst_dir=$inst_dir fod_dir=$fod_dir => inst_dir_is_fod_dir=$inst_dir_is_fod_dir venv_dir=$venv_dir static_dir=$static_dir" 1>&2
#exit
##
#############################################################################
#############################################################################
if [ "$install_basesw" = 1 ]; then
# requires ./install-centos-fixcentos-sqlite.sh in case of CENTOS7
echo "$0: step 1: installing base software dependencies (OS packages)" 1>&2
set -e
echo "Installing epel repo"
......@@ -99,126 +123,318 @@ if [ "$install_basesw" = 1 ]; then
yum-config-manager --enable remi
yum -q -y install redis
##
if [ "$try_fixup_for_old_os_sqlite" = 1 -a "$inside_docker" = 1 ]; then # only do try this fix inside docker env
if ! type -p sqlite3; then
echo "$0: sqlite3 not found, installing it" 1>&2
yum install -y sqlite3
fi
if [ "$try_fixup_for_old_os_sqlite" = 1 -a "$inside_docker" = 1 ]; then # only do try this fix inside docker env
. "./install-centos-fixcentos-sqlite.sh" "$fod_dir"
fi
fi
set +e
echo "$0: step 1 done" 1>&2
fi
##
echo "$0: step 1a: handling sqlite3 too old fixup post actions" 1>&2 # only needed for CENTOS
assume__sqlite_version__to_old=1
if [ "$use_old_django_version__autodetect" = 1 ]; then
sqlite_version="$(sqlite3 -version)"
if [ "$sqlite_version" != "${sqlite_version#3.7.}" ]; then
echo "$0: sqlite_version=$sqlite_version is too old, will use old Django version" 1>&2
assume__sqlite_version__to_old=1
else
echo "$0: sqlite_version=$sqlite_version is recent enough to use newer Django and dependencies" 1>&2
assume__sqlite_version__to_old=0
fi
fi
##
python_version="$(python3 --version | cut -d ' ' -f 2,2)"
if [ "$assume__sqlite_version__to_old" = 1 ]; then
echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2
cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt"
elif [ -e "$fod_dir/requirements.txt.python$python_version" ]; then
echo "$0: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2
cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
else
echo "$0: using $fod_dir/requirements.txt" 1>&2
fi
echo "$0: step 1b: preparing database system" 1>&2
#############################################################################
#############################################################################
if [ "$install_fodproper" = 0 ]; then
echo "$0: step 2a: installing Python dependencies only" 1>&2
set -e
echo "Setup partial python environment for FoD"
#virtualenv-3 /srv/venv
virtualenv-3 "$venv_dir"
if [ -x pyvenv ]; then
#pyvenv /srv/venv
pyvenv "$venv_dir"
else
#virtualenv-3 /srv/venv
virtualenv-3 "$venv_dir"
fi
ln -sf "$venv_dir" "$fod_dir/venv"
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
#fix for broken anyjson and cl
##
# fix for broken anyjson and cl
# TODO: fix this more cleanly
pip install 'setuptools<58'
cp -f requirements-centos.txt requirements.txt
pip install -r requirements.txt
# if [ "$assume__sqlite_version__to_old" = 1 ]; then
#
# # fix for broken anyjson and cl
# # TODO: fix this more cleanly
# #pip install 'setuptools<58'
#
# echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2
#
# cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt"
# else
#
# python_version="$(python3 --version | cut -d ' ' -f 1,1)"
# if [ -e "$fod_dir/requirements.txt.$python_version" ]; then
# echo "$0: using python version specific $fod_dir/requirements.txt.$python_version" 1>&2
# cp "$fod_dir/requirements.txt.$python_version" "$fod_dir/requirements.txt"
# fi
#
# fi
(cd "$fod_dir" && pip install -r requirements.txt)
echo "$0: step 2a done" 1>&2
else
echo "$0: step 2: installing FoD in installation dir + ensuring Python dependencies are installed + setting-up FoD settings, database preparations, and FoD run-time environment" 1>&2
set -e
echo "$0: step 2.0" 1>&2
echo "Setup python environment for FoD"
mkdir -p /var/log/fod "$venv_dir_base"
#virtualenv-3 /srv/venv
virtualenv-3 "$venv_dir"
##
echo "Setup python environment for FoD"
if [ -x pyvenv ]; then
#pyvenv /srv/venv
pyvenv "$venv_dir"
else
#virtualenv-3 /srv/venv
virtualenv-3 "$venv_dir"
fi
ln -sf "$venv_dir" "$fod_dir/venv"
(
set +e
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
#mkdir -p /srv/flowspy/log/
mkdir -p "$fod_dir/log/"
# Select source dir and copy FoD into /srv/flowspy/
if [ "`basename "$0"`" = install-centos.sh ]; then
# this script is in the source directory
#cp -f -r "`dirname $0`"/* /srv/flowspy/
cp -f -r "`dirname $0`"/* "$fod_dir"
elif [ -e /vagrant ]; then
# vagrant's copy in /vagrant/
#cp -f -r /vagrant/* /srv/flowspy/
cp -f -r /vagrant/* "$fod_dir"
elif [ -e ./install-centos.sh ]; then
# current directory is with the sourcecode
#cp -f -r ./* /srv/flowspy/
cp -f -r ./* "$fod_dir"
else
echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./"
exit 1
fi
set -e
#cd /srv/flowspy/
cd "$fod_dir"
(
cd flowspy
if [ -e settings.py.centos ]; then
cp -f settings.py.centos settings.py
elif [ ! -e settings.py ]; then
#cp -f settings.py.dist settings.py
cp -f settings.py.centos.dist settings.py
patch settings.py < settings.py.patch
sed -i "s#/srv/flowspy#$fod_dir#" "settings.py"
fi
)
cp -f requirements-centos.txt requirements.txt
##
pip install 'setuptools<58'
pip install -r requirements.txt
if [ ! -e "flowspy/settings_local.py" ]; then
touch flowspy/settings_local.py
fi
#./manage.py syncdb --noinput
#mkdir -p /srv/flowspy/static/
mkdir -p "$static_dir"
./manage.py collectstatic --noinput
./manage.py migrate
./manage.py loaddata initial_data
cp -f supervisord-centos.conf supervisord.conf
sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf"
cp -f runfod.centos.sh runfod.sh
mkdir -p /var/run/fod
#chown fod /var/run/fod
##
if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then
echo "adding pythonev wrapper" 1>&2
cat > "$fod_dir/pythonenv" <<EOF
set +e
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
##
#mkdir -p /srv/flowspy/log/
mkdir -p "$fod_dir/log/"
if true; then
echo "$0: step 2.1: coyping from source dir to installation dir $fod_dir" 1>&2
MYSELF="$(basename "$0")"
DIRNAME="$(dirname "$0")"
# Select source dir and copy FoD into /srv/flowspy/
if [ "$MYSELF" = "$SCRIPT_NAME" ]; then # if started as main script, e.g., in Docker or on OS-installation
# this script is in the source directory
#cp -f -r "`dirname $0`"/* /srv/flowspy/
#cp -f -r "$DIRNAME"/* "$fod_dir"
cp -f -r "$inst_dir"/* "$fod_dir"
elif [ -e /vagrant ]; then # running in vagrant with /vagrant available
# vagrant's copy in /vagrant/
#cp -f -r /vagrant/* /srv/flowspy/
cp -f -r /vagrant/* "$fod_dir"
elif [ -e "$SCRIPT_NAME" ]; then # running in vagrant with script current dir == install dir
# current directory is with the sourcecode
#cp -f -r ./* /srv/flowspy/
cp -f -r ./* "$fod_dir"
else
echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./"
exit 1
fi
fi
set -e
echo "$0: step 2.2: setting-up FoD settings" 1>&2
#cd /srv/flowspy/
cd "$fod_dir"
(
cd flowspy # jump into settings subdir flowspy
if [ "$inside_docker" = 1 -a -e settings.py.centos ]; then # user has own centos-specific settings prepared yet ?
cp -f settings.py.centos.docker settings.py
elif [ -e settings.py.centos ]; then # user has own centos-specific settings prepared yet ?
cp -f settings.py.centos settings.py
elif [ "$inside_docker" = 1 -a -e settings.py.docker ]; then # user has own settings prepared yet ?
cp -f settings.py.docker settings.py
elif [ -e settings.py ]; then # user has prepared a generic settings yet ?
: # nothing todo
else # prepare a settings.py from git repo's settings.py.dist/settings.py.centos.dist + settings.py.patch
if [ "$assume__sqlite_version__to_old" = 1 ]; then
echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old using settings.py.centos.dist" 1>&2
cp -f settings.py.centos.dist settings.py
else
cp -f settings.py.dist settings.py
fi
patch settings.py < settings.py.patch
sed -i "s#/srv/flowspy#$fod_dir#" "settings.py"
fi
)
if [ ! -e "flowspy/settings_local.py" ]; then
touch flowspy/settings_local.py
fi
echo "$0: step 2.3: ensuring Python dependencies are installed" 1>&2
if [ "$install_basesw" = 1 ]; then #are we running in --both mode, i.e. for the venv init is run for the first time, i.e. the problematic package having issues with to new setuptools is not yet installed?
# fix for broken anyjson and cl
# TODO: fix this more cleanly
pip install 'setuptools<58'
fi
# actual proper installation of python requirements
pip install -r requirements.txt
##
echo "$0: step 2.4: preparing FoD static files and database" 1>&2
echo "$0: step 2.4.1: preparing FoD static files" 1>&2
#mkdir -p /srv/flowspy/static/
mkdir -p "$static_dir"
([ ! -f "fodenv.sh" ] || source "./fodenv.sh"; ./manage.py collectstatic --noinput)
##
echo "$0: step 2.4.2.0: preparing DB and DB access" 1>&2
echo "$0: step 2.4.2.0: preparing FoD DB schema and basic data" 1>&2
echo "deploying/updating database schema" 1>&2
(
[ ! -f "fodenv.sh" ] || source "./fodenv.sh"
#./manage.py syncdb --noinput
#which sqlite3
#sqlite3 -version
#source "$venv_dir/bin/activate"
./manage.py migrate
./manage.py loaddata initial_data
)
echo 1>&2
##
echo "$0: step 2.5: preparing FoD run-time environment" 1>&2
echo "$0: step 2.5.1: preparing necessary dirs" 1>&2
mkdir -p /var/run/fod
#chown fod /var/run/fod
##
echo "$0: step 2.5.2: preparing FoD python wrapper" 1>&2
if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then
echo "adding pythonev wrapper" 1>&2
cat > "$fod_dir/pythonenv" <<EOF
#!/bin/bash
. "$venv_dir/bin/activate"
[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh"
exec "\$@"
EOF
chmod +x "$fod_dir/pythonenv"
echo 1>&2
fi
chmod +x "$fod_dir/pythonenv"
echo 1>&2
fi
##
echo "$0: step 2.5.3: preparing supervisord.conf" 1>&2
if [ "$assume__sqlite_version__to_old" = 1 -a -e "$fod_dir/supervisord-centos.conf" ]; then
echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using supervisord-centos.conf for old celery start syntax" 1>&2
cp -f "$fod_dir/supervisord-centos.conf" "$fod_dir/supervisord.conf"
elif [ -e "$fod_dir/supervisord.conf.dist" ]; then
echo "providing supervisord config" 1>&2
cp -f "$fod_dir/supervisord.conf.dist" "$fod_dir/supervisord.conf"
fi
sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf"
echo 1>&2
##
echo "$0: step 2.5.4: preparing runfod script" 1>&2
if [ "$assume__sqlite_version__to_old" = 1 ]; then
echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using runfod.centos.sh for old celery start syntax" 1>&2
cp -f runfod.centos.sh runfod.sh
fi
##
)
echo "$0: step 2 done" 1>&2
set +e
fi
#############################################################################
#############################################################################
install-debian.sh
+ 220
− 117
View file @ 7afcfe2f
#!/bin/bash
#!/bin/sh
#
# This script installs all dependencies for Firewall-on-Demand running in Python3
# with Celery, Redis, and sqlite.
#
SCRIPT_NAME="install-debian.sh"
fod_dir="/srv/flowspy"
venv_dir="/srv/venv"
......@@ -94,10 +97,11 @@ function conf_db_access () {
}
##
##############################################################################
##############################################################################
if [ -e "/.dockenv" ]; then
if [ -e "/.dockerenv" ]; then
echo "running inside docker assummed" 1>&2
inside_docker=1
fi
......@@ -206,7 +210,7 @@ echo "conf_db_access=$conf_db_access DB_NAME=$DB__FOD_DBNAME DB_USER=$DB__FOD_US
venv_dir_base="$(dirname "$venv_dir")"
static_dir="$fod_dir/static"
inst_dir="$(dirname "$0")"
mkdir -p "$fod_dir" || exit
......@@ -220,8 +224,13 @@ fi
echo "$0: inst_dir=$inst_dir fod_dir=$fod_dir => inst_dir_is_fod_dir=$inst_dir_is_fod_dir venv_dir=$venv_dir static_dir=$static_dir" 1>&2
#exit
#############################################################################
#############################################################################
if [ "$install_basesw" = 1 ]; then
echo "$0: step 1: installing base software dependencies (OS packages)" 1>&2
if [ "$install_db" = "" ]; then
mysql_server_pkg=("")
elif [ "$install_db" = "mysql" ]; then
......@@ -292,10 +301,16 @@ if [ -n "$install_db" ]; then
set +e
echo "$0: step 1 done" 1>&2
fi
##
echo "$0: step 1a: handling sqlite3 too old fixup post actions" 1>&2 # only needed for CENTOS
echo "$0: step 1b: preparing database system" 1>&2
if [ -n "$init_db" ]; then
set -e
......@@ -315,11 +330,27 @@ if [ -n "$init_db" ]; then
set +e
##
python_version="$(python3 --version | cut -d ' ' -f 2,2)"
#if [ "$assume__sqlite_version__to_old" = 1 ]; then
# echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2
# cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt"
if [ -e "$fod_dir/requirements.txt.python$python_version" ]; then
echo "$0: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2
cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
else
echo "$0: using $fod_dir/requirements.txt" 1>&2
fi
##
#############################################################################
#############################################################################
if [ "$install_fodproper" = 0 ]; then
echo "$0: step 2a: installing Python dependencies only" 1>&2
set -e
echo "Setup partial python environment for FoD"
......@@ -337,19 +368,35 @@ if [ "$install_fodproper" = 0 ]; then
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
# fix
pip install setuptools==57.5.0
##
# fix for broken anyjson and cl
# TODO: fix this more cleanly
pip install 'setuptools==57.5.0'
pip install wheel
pip install -r requirements.txt
else
echo "$0: step 2a done" 1>&2
else
echo "$0: step 2: installing FoD in installation dir + ensuring Python dependencies are installed + setting-up FoD settings, database preparations, and FoD run-time environment" 1>&2
set -e
echo "$0: step 2.0" 1>&2
id fod &>/dev/null || useradd -m fod
echo "Setup python environment for FoD"
#mkdir -p /var/log/fod /srv
mkdir -p /var/log/fod "$venv_dir_base"
##
echo "Setup python environment for FoD"
if [ -x pyvenv ]; then
#pyvenv /srv/venv
pyvenv "$venv_dir"
......@@ -358,70 +405,99 @@ else
virtualenv --python=python3 "$venv_dir"
fi
ln -sf "$venv_dir" "$fod_dir/venv"
(
set +e
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
#mkdir -p /srv/flowspy/
mkdir -p "$fod_dir"
if [ "$inst_dir_is_fod_dir" = 0 ]; then
# Select source dir and copy FoD into /srv/flowspy/
if [ "`basename "$0"`" = install-debian.sh ]; then
# this script is in the source directory
#cp -f -r "`dirname $0`"/* /srv/flowspy/
cp -f -r "$inst_dir"/* "$fod_dir"
elif [ -e /vagrant ]; then
# vagrant's copy in /vagrant/
#cp -f -r /vagrant/* /srv/flowspy/
cp -f -r /vagrant/* "$fod_dir"
elif [ -e ./install-centos.sh ]; then
# current directory is with the sourcecode
#cp -f -r ./* /srv/flowspy/
cp -f -r ./* "$fod_dir"
else
echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./"
exit 1
fi
set +e
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
fi
##
#find "$fod_dir/" -not -user fod -exec chown -v fod: {} \;
find "$fod_dir/" -not -user fod -exec chown fod: {} \;
set -e
#cd /srv/flowspy/
cd "$fod_dir"
(
cd flowspy
if [ ! -e settings.py ]; then
cp -f settings.py.dist settings.py
patch settings.py < settings.py.patch
sed -i "s#/srv/flowspy#$fod_dir#" "settings.py"
fi
)
if [ "$install_basesw" = 1 ]; then #are we running in --both mode, i.e. for the venv init is run for the first time, i.e. the problematic package having issues with to new setuptools is not yet installed?
# fix
pip install setuptools==57.5.0
:
fi
#mkdir -p /srv/flowspy/
mkdir -p "$fod_dir"
pip install -r requirements.txt
if [ "$inst_dir_is_fod_dir" = 0 ]; then
echo "$0: step 2.1: coyping from source dir to installation dir $fod_dir" 1>&2
MYSELF="$(basename "$0")"
# Select source dir and copy FoD into /srv/flowspy/
if [ "$MYSELF" = "$SCRIPT_NAME" ]; then # if started as main script, e.g., in Docker or on OS-installation
# this script is in the source directory
#cp -f -r "`dirname $0`"/* /srv/flowspy/
cp -f -r "$inst_dir"/* "$fod_dir"
elif [ -e /vagrant ]; then # running in vagrant with /vagrant available
# vagrant's copy in /vagrant/
#cp -f -r /vagrant/* /srv/flowspy/
cp -f -r /vagrant/* "$fod_dir"
elif [ -e "$SCRIPT_NAME" ]; then # running in vagrant with script current dir == install dir
# current directory is with the sourcecode
#cp -f -r ./* /srv/flowspy/
cp -f -r ./* "$fod_dir"
else
echo "Could not find FoD src directory tried `dirname $0`, /vagrant/, ./"
exit 1
fi
if [ ! -e "flowspy/settings_local.py" ]; then
touch flowspy/settings_local.py
fi
fi
#find "$fod_dir/" -not -user fod -exec chown -v fod: {} \;
find "$fod_dir/" -not -user fod -exec chown fod: {} \;
###
set -e
echo "$0: step 2.2: setting-up FoD settings" 1>&2
#cd /srv/flowspy/
cd "$fod_dir"
(
cd flowspy # jump into settings subdir flowspy
if [ "$inside_docker" = 1 -a -e settings.py.docker ]; then # user has own settings prepared yet ?
cp -f settings.py.docker settings.py
elif [ -e settings.py ]; then # user has prepared a generic settings yet ?
: # nothing todo
else # prepare from settings.py.dist + settings.py.patch
cp -f settings.py.dist settings.py
patch settings.py < settings.py.patch
sed -i "s#/srv/flowspy#$fod_dir#" "settings.py"
fi
)
if [ ! -e "flowspy/settings_local.py" ]; then
touch flowspy/settings_local.py
fi
echo "$0: step 2.3: ensuring Python dependencies are installed" 1>&2
if [ "$install_basesw" = 1 ]; then #are we running in --both mode, i.e. for the venv init is run for the first time, i.e. the problematic package having issues with to new setuptools is not yet installed?
# fix for broken anyjson and cl
# TODO: fix this more cleanly
pip install 'setuptools==57.5.0'
fi
# actual proper installation of python requirements
pip install -r requirements.txt
##
echo "$0: step 2.4.1: preparing log sub dirs" 1>&2
mkdir -p "$fod_dir/log" "$fod_dir/logs"
touch "$fod_dir/debug.log"
chown -R fod: "$fod_dir/log" "$fod_dir/logs" "$fod_dir/debug.log"
if [ "$try_install_docu" = 1 ]; then
echo "$0: step 2.4.2: compiling internal docu" 1>&2
echo "trying to install mkdocs-based documentation" 1>&2
(
set -e
......@@ -431,12 +507,19 @@ else
)
fi
#./manage.py syncdb --noinput
#mkdir -p /srv/flowspy/static/
mkdir -p "$static_dir"
./manage.py collectstatic --noinput
##
echo "$0: step 2.4: preparing FoD static files and database" 1>&2
echo "$0: step 2.4.1: preparing FoD static files" 1>&2
#mkdir -p /srv/flowspy/static/
mkdir -p "$static_dir"
([ ! -f "fodenv.sh" ] || source "./fodenv.sh"; ./manage.py collectstatic --noinput)
##
##
echo "$0: step 2.4.2.0: preparing DB and DB access" 1>&2
#if [ "$init_db" = "mariadb" -o "$init_db" = "mysql" ]; then
# init_mysqllikedb "$DB__FOD_DBNAME" "$DB__FOD_USER" "$DB__FOD_PASSWORD"
......@@ -448,84 +531,104 @@ else
echo 1>&2
fi
##
##
echo "deploying/updating database schema" 1>&2
./manage.py migrate
./manage.py loaddata initial_data
echo 1>&2
#
echo "$0: step 2.4.2.0: preparing FoD DB schema and basic data" 1>&2
# ./manage.py aboove may have created debug.log with root permissions:
chown -R fod: "$fod_dir/log" "$fod_dir/logs" "$fod_dir/debug.log"
[ ! -d "/var/log/fod" ] || chown -R fod: "/var/log/fod"
echo "deploying/updating database schema" 1>&2
(
[ ! -f "fodenv.sh" ] || source "./fodenv.sh"
#./manage.py syncdb --noinput
./manage.py migrate
./manage.py loaddata initial_data
)
echo 1>&2
#
##
echo "providing supervisord config" 1>&2
cp -f "$fod_dir/supervisord.conf.dist" "$fod_dir/supervisord.conf"
sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf"
echo 1>&2
# ./manage.py above may have created debug.log with root permissions:
chown -R fod: "$fod_dir/log" "$fod_dir/logs" "$fod_dir/debug.log"
[ ! -d "/var/log/fod" ] || chown -R fod: "/var/log/fod"
#
#
echo "$0: step 2.5: preparing FoD run-time environment" 1>&2
mkdir -p /var/run/fod
chown fod: /var/run/fod
echo "$0: step 2.5.1: preparing necessary dirs" 1>&2
#
mkdir -p /var/run/fod
chown fod: /var/run/fod
##
##
if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then
echo "adding pythonev wrapper" 1>&2
cat > "$fod_dir/pythonenv" <<EOF
echo "$0: step 2.5.2: preparing FoD python wrapper" 1>&2
if [ "$ensure_installed_pythonenv_wrapper" = 1 -a \( "$inside_docker" = 1 -o ! -e "$fod_dir/pythonenv" \) ]; then
echo "adding pythonev wrapper" 1>&2
cat > "$fod_dir/pythonenv" <<EOF
#!/bin/bash
. "$venv_dir/bin/activate"
[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh"
exec "\$@"
EOF
chmod +x "$fod_dir/pythonenv"
echo 1>&2
fi
chmod +x "$fod_dir/pythonenv"
echo 1>&2
fi
##
##
fod_systemd_dir="$fod_dir/systemd"
cp -f "$fod_systemd_dir/fod-gunicorn.service.dist" "$fod_systemd_dir/fod-gunicorn.service"
sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-gunicorn.service"
echo "$0: step 2.5.3: preparing supervisord.conf" 1>&2
cp -f "$fod_systemd_dir/fod-celeryd.service.dist" "$fod_systemd_dir/fod-celeryd.service"
sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-celeryd.service"
cp -f "$fod_dir/supervisord.conf.dist" "$fod_dir/supervisord.conf"
sed -i "s#/srv/flowspy#$fod_dir#" "$fod_dir/supervisord.conf"
echo 1>&2
cp -f "$fod_systemd_dir/fod-status-email-user@.service.dist" "$fod_systemd_dir/fod-status-email-user@.service"
sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-status-email-user@.service"
##
if [ "$install_systemd_services" = 1 ]; then
echo 1>&2
echo "Installing systemd services" 1>&2
echo 1>&2
#cp -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "/etc/systemd/system/"
cp -v -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "$fod_systemd_dir/fod-status-email-user@.service" "/etc/systemd/system/" 1>&2
systemctl daemon-reload
echo "$0: step 2.5.5: preparing systemd files" 1>&2
systemctl enable fod-gunicorn
systemctl enable fod-celeryd
fod_systemd_dir="$fod_dir/systemd"
cp -f "$fod_systemd_dir/fod-gunicorn.service.dist" "$fod_systemd_dir/fod-gunicorn.service"
sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-gunicorn.service"
systemctl restart fod-gunicorn
systemctl restart fod-celeryd
cp -f "$fod_systemd_dir/fod-celeryd.service.dist" "$fod_systemd_dir/fod-celeryd.service"
sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-celeryd.service"
sleep 5
SYSTEMD_COLORS=1 systemctl status fod-gunicorn | cat
echo
SYSTEMD_COLORS=1 systemctl status fod-celeryd | cat
echo
cp -f "$fod_systemd_dir/fod-status-email-user@.service.dist" "$fod_systemd_dir/fod-status-email-user@.service"
sed -i "s#/srv/flowspy#$fod_dir#g" "$fod_systemd_dir/fod-status-email-user@.service"
fi
if [ "$install_systemd_services" = 1 ]; then
echo 1>&2
echo "Installing systemd services" 1>&2
echo 1>&2
#cp -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "/etc/systemd/system/"
cp -v -f "$fod_systemd_dir/fod-gunicorn.service" "$fod_systemd_dir/fod-celeryd.service" "$fod_systemd_dir/fod-status-email-user@.service" "/etc/systemd/system/" 1>&2
systemctl daemon-reload
systemctl enable fod-gunicorn
systemctl enable fod-celeryd
systemctl restart fod-gunicorn
systemctl restart fod-celeryd
sleep 5
SYSTEMD_COLORS=1 systemctl status fod-gunicorn | cat
echo
SYSTEMD_COLORS=1 systemctl status fod-celeryd | cat
echo
fi
)
echo "$0: step 2 done" 1>&2
set +e
fi
#############################################################################
#############################################################################
requirements.txt.python3.6.8 0 → 100644
+ 42
− 0
View file @ 7afcfe2f
setuptools==59.1.1
argparse
cl
celery==5.2.0b3
django-admin
django-picklefield==2.0
django-registration==3.1.2
djangorestframework==3.12.0
django-tinymce
django-form-utils
django-settings-export
Django==2.2.28
gevent
greenlet
gunicorn
kombu==5.2.0rc1
lxml
mailer
mysqlclient
ncclient
paramiko
pycrypto
pyparsing
python-dateutil
python-memcached
PyYAML
#wsgiref
six
requests
pysnmp
ply
ipaddress
idna
appdirs
anyjson
cryptography
redis
pytest==6.2.5
pytest-django==4.4.0
psutil
longerusername
intervaltree
runfod-supervisord.sh
+ 18
− 3
View file @ 7afcfe2f
......@@ -40,7 +40,12 @@ while [ $# -gt 0 ]; do
done
##
##############################################################################
##############################################################################
if [ -e "$fod_dir/fodenv.sh" ]; then
. "$fod_dir/fodenv.sh"
fi
#. /srv/venv/bin/activate
. "$venv_dir/bin/activate"
......@@ -51,6 +56,7 @@ if [ ! -e "$fod_dir/pythonenv" ]; then
cat > "$fod_dir/pythonenv" <<EOF
#!/bin/bash
. "$venv_dir/bin/activate"
[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh"
exec "\$@"
EOF
#chmod +x /srv/flowspy/pythonenv
......@@ -102,7 +108,7 @@ systemctl disable redis
#useradd -m fod
(
set -x
#set -x
#source /srv/venv/bin/activate
source "$venv_dir/bin/activate"
#cd /srv/flowspy && ./manage.py collectstatic --noinput
......@@ -113,7 +119,13 @@ systemctl disable redis
sysctl vm.overcommit_memory=1
# supervisord.conf
cp -f supervisord.conf /etc
if [ -f supervisord.conf.prep ]; then
echo "$0: using supervisord.conf.prep" 1>&2
cp -f supervisord.conf.prep /etc/supervisord.conf
else
echo "$0: using supervisord.conf" 1>&2
cp -f supervisord.conf /etc
fi
mkdir -p /var/run/fod /var/log/fod
......@@ -128,3 +140,6 @@ chown fod "$fod_dir" "$fod_dir/log" "$fod_dir/logs" "$fod_dir/log/"* "$fod_dir/l
mkdir -p /var/run/supervisor
exec /usr/bin/supervisord -n -c /etc/supervisord.conf
##############################################################################
##############################################################################
runfod.centos.sh
+ 12
− 0
View file @ 7afcfe2f
......@@ -47,17 +47,27 @@ while [ $# -gt 0 ]; do
done
##############################################################################
##############################################################################
if [ -e "$fod_dir/fodenv.sh" ]; then
. "$fod_dir/fodenv.sh"
fi
##
#. /srv/venv/bin/activate
. "$venv_dir/bin/activate"
##
#if [ ! -e /srv/flowspy/pythonenv ]; then
if [ ! -e "$fod_dir/pythonenv" ]; then
#cat > /srv/flowspy/pythonenv <<EOF
cat > "$fod_dir/pythonenv" <<EOF
#!/bin/bash
. "$venv_dir/bin/activate"
[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh"
exec "\$@"
EOF
#chmod +x /srv/flowspy/pythonenv
......@@ -93,4 +103,6 @@ echo "Starting FoD gunicorn in foreground" 1>&2
#exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi -w 1 -k gevent --limit-request-fields 10000 --timeout 30 #--preload
exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi --reload -w 1 -k gevent --limit-request-fields 10000 --timeout 30
##############################################################################
##############################################################################
runfod.sh
+ 12
− 0
View file @ 7afcfe2f
......@@ -47,17 +47,27 @@ while [ $# -gt 0 ]; do
done
##############################################################################
##############################################################################
if [ -e "$fod_dir/fodenv.sh" ]; then
. "$fod_dir/fodenv.sh"
fi
##
#. /srv/venv/bin/activate
. "$venv_dir/bin/activate"
##
#if [ ! -e /srv/flowspy/pythonenv ]; then
if [ ! -e "$fod_dir/pythonenv" ]; then
#cat > /srv/flowspy/pythonenv <<EOF
cat > "$fod_dir/pythonenv" <<EOF
#!/bin/bash
. "$venv_dir/bin/activate"
[ ! -e "$fod_dir/fodenv.sh" ] || . "$fod_dir/fodenv.sh"
exec "\$@"
EOF
#chmod +x /srv/flowspy/pythonenv
......@@ -93,4 +103,6 @@ echo "Starting FoD gunicorn in foreground" 1>&2
#exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi -w 1 -k gevent --limit-request-fields 10000 --timeout 30 #--preload
exec gunicorn -b 0.0.0.0:8000 flowspy.wsgi --reload -w 1 -k gevent --limit-request-fields 10000 --timeout 30
##############################################################################
##############################################################################
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment