Skip to content
Snippets Groups Projects
Commit ecc65ff8 authored by David Schmitz's avatar David Schmitz
Browse files

Merge branch 'feature/exabgp_support2' of github.com:GEANT/FOD into feature/exabgp_support2

parents f53dc935 eed54596
No related branches found
No related tags found
No related merge requests found
Showing
with 450 additions and 20 deletions
= all-in-one-build of NeMo-detection+mitigation container and Freertr router + test hosts
pre-requisites: docker, ethtool
1.) clone all nemo-repos as subdirs here (including ./nemo-docker)
2.) run ./mynemo-docker-dind --allx
(will build and start everything;
finally will run vsmd process in foreground (in vsmd1 inner container); Please keep it running!
(TODO: run vsmd in background as a daemon controlled by systemd in vsmd1 inner container) )
3.) (in other terminal:) run ./mynemo-docker-dind --nemo-get-filename-of-certfile # get location of HTTPS cert file (accessible as root, to be imported into your web browser)
4.) access localhost port 443 via your web browser, with the HTTPS certfile from 3. installed
(if you need to re-install a new version of this HTTPS certificate, do not forget to retract the previous version)
If the terminal where vsmd runs in foreground is closed for any reason, or vsmd therein is stopped or so,
you can restart vsmd in foreground again by:
./mynemo-docker-dind --run-vsmd-in-vsmd-container
...
========
= build+run nemo docker-in-docker container together with freertr+attack/victim-hosts docker-compose
0)
......
......@@ -60,6 +60,7 @@ fi
#xarg ./mynemo-docker-dind --docker.containerids # get inside container ids
#xarg
#xarg ./mynemo-docker-dind --setup-and-run-vsmd-container # setup and run vsmd inner container; finally run vsmd in foreground
#xarg ./mynemo-docker-dind --run-vsmd-in-vsmd-container # re-run vsmd in foreground in vsmd inner container
#xarg
#xarg 3. related to freertr+host1+host2 docker-compose:
#xarg
......@@ -80,6 +81,8 @@ docker_outer_volume_forinnerdockerdir="dind1"
outer_https_port="8000"
extra_port_redirection_args=(-p "8025:8025")
docker_log_postfix="_1" # has to be consistent with docker_outer__container_name?
#
......@@ -210,7 +213,7 @@ elif [ "$1" = "--docker-build" ]; then #arg
##
echo "# starting new debian-docker-in-docker container:" 1>&2
(set -x; docker run -d --privileged --volume "$SSH_AUTH_SOCK:/ssh-auth-sock" --volume "$docker_outer_volume_forinnerdockerdir:/var/lib/docker/" --hostname "$docker_outer_hostname" --name "$docker_outer__container_name" -t -p "$outer_https_port:443" "$docker_outer__image_name")
(set -x; docker run -d --privileged --volume "$SSH_AUTH_SOCK:/ssh-auth-sock" --volume "$docker_outer_volume_forinnerdockerdir:/var/lib/docker/" --hostname "$docker_outer_hostname" --name "$docker_outer__container_name" -t -p "$outer_https_port:443" "${extra_port_redirection_args[@]}" "$docker_outer__image_name")
#(set -x; docker run -d --privileged --net dind-mgmt --net nemo-flow-input --volume "$SSH_AUTH_SOCK:/ssh-auth-sock" --volume "$docker_outer_volume_forinnerdockerdir:/var/lib/docker/" --hostname "$docker_outer_hostname" --name "$docker_outer__container_name" -t -p 8000:443 "$docker_outer__image_name")
#(set -x; docker run -d --privileged --net dind-mgmt --volume "$SSH_AUTH_SOCK:/ssh-auth-sock" --volume "$docker_outer_volume_forinnerdockerdir:/var/lib/docker/" --hostname "$docker_outer_hostname" --name "$docker_outer__container_name" -t -p 8000:443 "$docker_outer__image_name")
docker network connect "$network1_name" "$docker_outer__container_name"
......@@ -509,13 +512,15 @@ elif [ "$1" = "--all1" ]; then #arg # --all --prune --verbose
shift 1
#exec "$0" --all --prune --verbose "$@" #arg rest
exec "$0" --build-and-setup --prune --verbose "$@" #arg rest
#exec "$0" --build-and-setup --prune --verbose "$@" #arg rest
exec "$0" --build-and-setup --prune --verbose --dev "$@" #arg rest
elif [ "$1" = "--all0" ]; then #arg # --all --prune --verbose
shift 1
#exec "$0" --build-and-bash --build-without-data -c './mygit_clone_nemo0 && cd nemo-docker && ./setup.sh "$@"' -- "$@" #arg rest
exec "$0" --build-and-setup --build-without-data --prune --pre-setup-shcode "./mygit_clone_nemo0" --verbose
#exec "$0" --build-and-setup --build-without-data --prune --pre-setup-shcode "./mygit_clone_nemo0" --verbose
exec "$0" --build-and-setup --build-without-data --prune --pre-setup-shcode "./mygit_clone_nemo0" --verbose --dev
#arg
......@@ -526,6 +531,11 @@ elif [ "$1" = "--setup-and-run-vsmd-container" ]; then #arg # prep vsmd inner do
#"$0" --exec ./mynemo-mitigation-vsmd-install
"$0" --exec ./mynemo-mitigation-init-container.sh
elif [ "$1" = "--run-vsmd-in-vsmd-container" ]; then #arg # just run vsmd in vsmd continer in foreground
shift 1
"$0" --exec ./mynemo-mitigation-vsmd-run-in-container
#arg
elif [ "$1" = "--freertr-testnet-prep-and-run" ]; then #arg # prep and run docker-compose for freertr+host1+host2 (started in back-ground)
......@@ -757,7 +767,7 @@ elif [ "$1" = "--allx" ]; then #arg # all-in-one mode
set -x
# build and init nemo-outer container and build and init and run all detection container inside
"$0" --all1
"$0" --all1 "$@"
# init frnet comprising freertr router and host1/host2 test network
"$0" --freertr-testnet-prep-and-run
......
#!/bin/bash
#
##
container_name="$1"
shift 1
[ -n "$container_name" ] || container_name="vsmd1"
##
# assumes an initialized and running inner container, having been setup by ./mynemo-mitigation-init-container.sh before
# just re-runs last action of ./mynemo-mitigation-init-container.sh, which is to run vsmd process in foreground
#docker exec -ti "$container_name" ./mynemo-mitigation-vsmd-install-and-run
docker exec -ti "$container_name" ./mynemo-mitigation-vsmd-run
......@@ -91,9 +91,31 @@ for item in fileinput.input():
afi = "IPv4"
dest_net = line[14]
source_net = line[11]
add_opts_str=""
proto = line[10].split("-")[0]
sport = line[13].split("-")[0]
proto = proto.rstrip()
if proto=="all":
proto=""
if proto!="":
add_opts_str=add_opts_str+",Proto:="+str(proto)
dport = line[16].split("-")[0]
dport = dport.rstrip()
if dport=="all":
dport=""
if dport!="":
print("using dport="+str(dport)+".")
add_opts_str=add_opts_str+",DPort:="+str(dport)
sport = line[13].split("-")[0]
sport = sport.rstrip()
if sport=="all":
sport=""
if sport!="":
add_opts_str=add_opts_str+",SPort:="+str(sport)
match = line[7].split("=")[1].replace(")", "")
match_left = match.split("(")[0]
match_right = match.split("(")[1]
......@@ -117,7 +139,8 @@ for item in fileinput.input():
match_left = match_left,
match_right = match_right,
drop_left = drop_left,
drop_right = drop_right
drop_right = drop_right,
add_opts_str = add_opts_str
)
#print("content="+str(content))
......
AFI: {{ afi }}
Flow :Dest:{{ dest_net }}/{{ dest_mask }},Source:{{ source_net }}/{{ source_mask }},Proto:={{ proto }},DPort:={{ dport }},SPort:={{ sport }}
Flow :Dest:{{ dest_net }}/{{ dest_mask }},Source:{{ source_net }}/{{ source_mask }}{{ add_opts_str }}
Actions : discard
Synced: TRUE
Last Error: 0:No error
......
......@@ -502,10 +502,12 @@ fi
python_version="$(python3 --version | cut -d ' ' -f 2,2)"
if [ "$assume__sqlite_version__to_old" = 1 ]; then
echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2
cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt"
#cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt"
cp "$fod_dir/requirements.txt.d/requirements-centos.txt" "$fod_dir/requirements.txt"
elif [ -e "$fod_dir/requirements.txt.python$python_version" ]; then
echo "$0: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2
cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
#cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
cp "$fod_dir/requirements.txt.d/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
else
echo "$0: using $fod_dir/requirements.txt" 1>&2
fi
......
......@@ -48,6 +48,8 @@ fi
#############################################################################
#############################################################################
show_env_for_debugging=0
fod_dir="/srv/flowspy"
venv_dir="/srv/venv"
......@@ -148,6 +150,11 @@ ifc_setup__wait_for_ifc__in_runfod=0
findfix_file_permissions=1
#
requirements_txt__filename_relative__default="requirements.txt"
requirements_txt__filename_relative="$requirements_txt__filename_relative__default"
##############################################################################
##############################################################################
......@@ -332,7 +339,7 @@ while [ $# -gt 0 ]; do
install_basesw_os=1
#install_basesw_python=0
#install_fodproper=0
elif [ $# -ge 1 -a "$1" = "--basesw_python" ]; then
elif [ $# -ge 1 -a \( "$1" = "--basesw_python" -o "$1" = "--python-update-only" \) ]; then
shift 1
install_default_used=0
......@@ -353,6 +360,13 @@ while [ $# -gt 0 ]; do
#install_basesw_os=0
#install_basesw_python=0
install_fodproper=1
elif [ $# -ge 1 -a \( "$1" = "--pyrequirements" \) ]; then
shift 1
requirements_txt__filename_relative="$1"
shift 1
elif [ "$1" = "--show-env" ]; then
shift 1
show_env_for_debugging=1
elif [ $# -ge 1 -a \( "$1" = "--supervisor" -o "$1" = "--supervisord" \) ]; then
shift 1
install_with_supervisord=1
......@@ -685,11 +699,16 @@ python_version="$(python3 --version | cut -d ' ' -f 2,2)"
#if [ "$assume__sqlite_version__to_old" = 1 ]; then
# echo "$0: assume__sqlite_version__to_old=$assume__sqlite_version__to_old => using requirements-centos.txt" 1>&2
# cp "$fod_dir/requirements-centos.txt" "$fod_dir/requirements.txt"
if [ -e "$fod_dir/requirements.txt.python$python_version" ]; then
echo "$0: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2
cp "$fod_dir/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
if [ "$requirements_txt__filename_relative" != "$requirements_txt__filename_relative__default" ]; then
echo "python reqs: using $fod_dir/$requirements_txt__filename_relative" 1>&2
cp -f "$fod_dir/$requirements_txt__filename_relative" "$fod_dir/requirements.txt"
else
echo "$0: using $fod_dir/requirements.txt" 1>&2
if [ -e "$fod_dir/requirements.txt.python$python_version" ]; then
echo "python reqs: using python version specific $fod_dir/requirements.txt.python$python_version" 1>&2
cp -f "$fod_dir/requirements.txt.d/requirements.txt.python$python_version" "$fod_dir/requirements.txt"
else
echo "python reqs: using default $fod_dir/$requirements_txt__filename_relative__default" 1>&2
fi
fi
#############################################################################
......@@ -800,7 +819,9 @@ if [ "$install_fodproper" = 1 ]; then
source "$venv_dir/bin/activate"
export 1>&2
if [ "$show_env_for_debugging" = 1 ]; then
export 1>&2
fi
##
......
......@@ -27,7 +27,7 @@ PyYAML
#wsgiref
six
requests
pysnmp
pysnmp<=5.1.0
ply
ipaddress
idna
......
amqp==5.2.0
anyjson==0.3.3
appdirs==1.4.4
asgiref==3.8.1
async-timeout==4.0.3
attrs==24.2.0
bcrypt==4.2.0
billiard==3.6.4.0
celery==5.2.3
certifi==2024.8.30
cffi==1.17.1
charset-normalizer==3.4.0
cl==0.0.3
click==8.1.7
click-didyoumean==0.3.1
click-plugins==1.1.1
click-repl==0.3.0
confusable-homoglyphs==3.3.1
cryptography==43.0.1
Django==3.2.16
django-admin==2.0.2
django-excel-response2==3.0.6
django-form-utils==1.0.3
django-picklefield==2.0
django-registration==3.1.2
django-settings-export==1.2.1
django-six==1.0.5
django-tinymce==4.1.0
djangorestframework==3.12.0
excel-base==1.0.4
gevent==24.10.2
greenlet==3.1.1
gunicorn==23.0.0
idna==3.10
iniconfig==2.0.0
intervaltree==3.1.0
ipaddress==1.0.23
isoweek==1.3.3
kombu==5.2.3
longerusername==0.4
lxml==5.3.0
mailer==0.8.1
mysqlclient==2.2.4
ncclient==0.6.16
packaging==24.1
paramiko==3.5.0
pluggy==1.5.0
ply==3.11
prompt_toolkit==3.0.48
psutil==6.1.0
py==1.11.0
pyasn1==0.4.8
pycparser==2.22
pycrypto==2.6.1
PyNaCl==1.5.0
pyparsing==3.2.0
pysmi==1.2.1
pysnmp==5.1.0
pysnmpcrypto==0.0.4
pytest==6.2.5
pytest-django==4.4.0
python-dateutil==2.9.0.post0
python-memcached==1.62
pytz==2024.2
PyYAML==6.0.2
redis==5.1.1
requests==2.32.3
screen==1.0.1
six==1.16.0
sortedcontainers==2.4.0
sqlparse==0.5.1
TimeConvert==3.0.13
toml==0.10.2
typing_extensions==4.12.2
tzlocal==5.2
urllib3==2.2.3
vine==5.1.0
wcwidth==0.2.13
xlwt==1.3.0
zope.event==5.0
zope.interface==7.1.0
amqp==5.2.0
anyjson==0.3.3
appdirs==1.4.4
asgiref==3.8.1
async-timeout==4.0.3
attrs==24.2.0
backports.zoneinfo==0.2.1
bcrypt==4.2.0
billiard==3.6.4.0
celery==5.2.3
certifi==2024.8.30
cffi==1.17.1
charset-normalizer==3.4.0
cl==0.0.3
click==8.1.7
click-didyoumean==0.3.1
click-plugins==1.1.1
click-repl==0.3.0
confusable-homoglyphs==3.3.1
cryptography==43.0.3
Django==3.2.16
django-admin==2.0.2
django-excel-response2==3.0.6
django-form-utils==1.0.3
django-picklefield==2.0
django-registration==3.1.2
django-settings-export==1.2.1
django-six==1.0.5
django-tinymce==4.1.0
djangorestframework==3.12.0
excel-base==1.0.4
gevent==24.2.1
greenlet==3.1.1
gunicorn==23.0.0
idna==3.10
iniconfig==2.0.0
intervaltree==3.1.0
ipaddress==1.0.23
isoweek==1.3.3
kombu==5.2.3
longerusername==0.4
lxml==5.3.0
mailer==0.8.1
mysqlclient==2.2.5
ncclient==0.6.16
packaging==24.1
paramiko==3.5.0
pluggy==1.5.0
ply==3.11
prompt-toolkit==3.0.48
psutil==6.1.0
py==1.11.0
pyasn1==0.4.8
pycparser==2.22
pycrypto==2.6.1
PyNaCl==1.5.0
pyparsing==3.1.4
pysmi==1.2.1
pysnmp==5.1.0
pysnmpcrypto==0.0.4
pytest==6.2.5
pytest-django==4.4.0
python-dateutil==2.9.0.post0
python-memcached==1.62
pytz==2024.2
PyYAML==6.0.2
redis==5.1.1
requests==2.32.3
screen==1.0.1
six==1.16.0
sortedcontainers==2.4.0
sqlparse==0.5.1
TimeConvert==3.0.13
toml==0.10.2
typing-extensions==4.12.2
tzlocal==5.2
urllib3==2.2.3
vine==5.1.0
wcwidth==0.2.13
xlwt==1.3.0
zope.event==5.0
zope.interface==7.1.0
amqp==5.2.0
anyjson==0.3.3
appdirs==1.4.4
asgiref==3.8.1
async-timeout==4.0.3
attrs==24.2.0
bcrypt==4.2.0
billiard==3.6.4.0
celery==5.2.3
certifi==2024.8.30
cffi==1.17.1
charset-normalizer==3.4.0
cl==0.0.3
click==8.1.7
click-didyoumean==0.3.1
click-plugins==1.1.1
click-repl==0.3.0
confusable-homoglyphs==3.3.1
cryptography==43.0.1
Django==3.2.16
django-admin==2.0.2
django-excel-response2==3.0.6
django-form-utils==1.0.3
django-picklefield==2.0
django-registration==3.1.2
django-settings-export==1.2.1
django-six==1.0.5
django-tinymce==4.1.0
djangorestframework==3.12.0
excel-base==1.0.4
gevent==24.10.2
greenlet==3.1.1
gunicorn==23.0.0
idna==3.10
iniconfig==2.0.0
intervaltree==3.1.0
ipaddress==1.0.23
isoweek==1.3.3
kombu==5.2.3
longerusername==0.4
lxml==5.3.0
mailer==0.8.1
mysqlclient==2.2.4
ncclient==0.6.16
packaging==24.1
paramiko==3.5.0
pluggy==1.5.0
ply==3.11
prompt_toolkit==3.0.48
psutil==6.1.0
py==1.11.0
pyasn1==0.4.8
pycparser==2.22
pycrypto==2.6.1
PyNaCl==1.5.0
pyparsing==3.2.0
pysmi==1.2.1
pysnmp==5.1.0
pysnmpcrypto==0.0.4
pytest==6.2.5
pytest-django==4.4.0
python-dateutil==2.9.0.post0
python-memcached==1.62
pytz==2024.2
PyYAML==6.0.2
redis==5.1.1
requests==2.32.3
screen==1.0.1
six==1.16.0
sortedcontainers==2.4.0
sqlparse==0.5.1
TimeConvert==3.0.13
toml==0.10.2
typing_extensions==4.12.2
tzlocal==5.2
urllib3==2.2.3
vine==5.1.0
wcwidth==0.2.13
xlwt==1.3.0
zope.event==5.0
zope.interface==7.1.0
amqp==5.2.0
anyjson==0.3.3
appdirs==1.4.4
asgiref==3.8.1
async-timeout==4.0.3
attrs==24.2.0
backports.zoneinfo==0.2.1
bcrypt==4.2.0
billiard==3.6.4.0
celery==5.2.3
certifi==2024.8.30
cffi==1.17.1
charset-normalizer==3.4.0
cl==0.0.3
click==8.1.7
click-didyoumean==0.3.1
click-plugins==1.1.1
click-repl==0.3.0
confusable-homoglyphs==3.3.1
cryptography==43.0.3
Django==3.2.16
django-admin==2.0.2
django-excel-response2==3.0.6
django-form-utils==1.0.3
django-picklefield==2.0
django-registration==3.1.2
django-settings-export==1.2.1
django-six==1.0.5
django-tinymce==4.1.0
djangorestframework==3.12.0
excel-base==1.0.4
gevent==24.2.1
greenlet==3.1.1
gunicorn==23.0.0
idna==3.10
iniconfig==2.0.0
intervaltree==3.1.0
ipaddress==1.0.23
isoweek==1.3.3
kombu==5.2.3
longerusername==0.4
lxml==5.3.0
mailer==0.8.1
mysqlclient==2.2.5
ncclient==0.6.16
packaging==24.1
paramiko==3.5.0
pluggy==1.5.0
ply==3.11
prompt-toolkit==3.0.48
psutil==6.1.0
py==1.11.0
pyasn1==0.4.8
pycparser==2.22
pycrypto==2.6.1
PyNaCl==1.5.0
pyparsing==3.1.4
pysmi==1.2.1
pysnmp==5.1.0
pysnmpcrypto==0.0.4
pytest==6.2.5
pytest-django==4.4.0
python-dateutil==2.9.0.post0
python-memcached==1.62
pytz==2024.2
PyYAML==6.0.2
redis==5.1.1
requests==2.32.3
screen==1.0.1
six==1.16.0
sortedcontainers==2.4.0
sqlparse==0.5.1
TimeConvert==3.0.13
toml==0.10.2
typing-extensions==4.12.2
tzlocal==5.2
urllib3==2.2.3
vine==5.1.0
wcwidth==0.2.13
xlwt==1.3.0
zope.event==5.0
zope.interface==7.1.0
......@@ -17,11 +17,16 @@ WorkingDirectory=/srv/flowspy
#RuntimeDirectory=yourservice
#RuntimeDirectoryMode=0755
ExecStartPre=-/bin/mkdir -p /var/run/fod/
ExecStartPre=-/bin/chown fod: /var/run/fod/
ExecStartPre=-/bin/rm -f -r /var/run/fod/snmppoll.lock
ExecStartPre=-/bin/rm -f /srv/flowspy/snmp_temp_data.lock
# executed as root:
#PermissionsStartOnly=True
ExecStartPre=+-/bin/mkdir -p /var/run/fod/
ExecStartPre=+-/bin/chown fod: /var/run/fod/
ExecStartPre=+-/bin/rm -f -r /var/run/fod/snmppoll.lock
ExecStartPre=+-/bin/rm -f /srv/flowspy/snmp_temp_data.lock
# executed as user fod:
ExecStartPre=-/srv/flowspy/flowspy/run_celery_preactions
#ExecStart=/srv/flowspy/pythonenv celery -A flowspy worker -B --concurrency=2 -l debug -f /srv/flowspy/log/celery.log
ExecStart=/srv/flowspy/pythonenv celery -A flowspy worker -B --concurrency=2 -l info -f /srv/flowspy/log/celery.log
#PIDFile=/tmp/yourservice.pid
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment