diff --git a/inventories/filebeat b/inventories/filebeat index 7021899acef7bfcbef3ea5d72fe9041fe14cd660..9f4153ac2aa99966fc0a950156f51fe596d69065 100644 --- a/inventories/filebeat +++ b/inventories/filebeat @@ -1,7 +1,7 @@ [filebeat] -soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" -soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" -soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text" #soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="text" #soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text" diff --git a/roles/build/files/nifisupervisord.conf b/roles/build/files/nifisupervisord.conf new file mode 100644 index 0000000000000000000000000000000000000000..dcfa15c6e07538c492047eda004bfaec585237a1 --- /dev/null +++ b/roles/build/files/nifisupervisord.conf @@ -0,0 +1,44 @@ +[unix_http_server] +file=/tmp/supervisor.sock + +[supervisord] +pidfile=/tmp/supervisord.pid +nodaemon=true +logfile=/var/log/supervisor/supervisord.log +logfile_maxbytes=10MB +logfile_backups=10 +loglevel=info +childlogdir=/var/log/supervisor/ + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock + +[program:nifi] +directory=/opt/nifi/nifi-current +user=nifi +group=nifi +command=/bin/bash -c '/opt/nifi/nifi-current/bin/nifi.sh run' +autostart=false +autorestart=true +logfile_maxbytes=10MB +stdout_logfile_backups = 0 +stderr_logfile_backups = 0 +stderr_logfile = /var/log/supervisor/nifi_stderr.log +stdout_logfile = /var/log/supervisor/nifi_stdout.log + + +[program:filebeat] +directory=/opt/filebeat +user=root +group=root +command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml' +autostart=false +autorestart=true +logfile_maxbytes=10MB +stdout_logfile_backups = 0 +stderr_logfile_backups = 0 +stderr_logfile = /var/log/supervisor/filebeat_stderr.log +stdout_logfile = /var/log/supervisor/filebeat_stdout.log diff --git a/roles/build/templates/nifi/Dockerfile.j2 b/roles/build/templates/nifi/Dockerfile.j2 index 36855e509447796e5901d337eaceb044250ba957..916c96c42771ba4a76a7c767a8521566862041c1 100644 --- a/roles/build/templates/nifi/Dockerfile.j2 +++ b/roles/build/templates/nifi/Dockerfile.j2 @@ -96,4 +96,8 @@ WORKDIR ${NIFI_HOME} # Also we need to use relative path, because the exec form does not invoke a command shell, # thus normal shell processing does not happen: # https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example -ENTRYPOINT ["/bin/bash"] +USER root +RUN yum install -y supervisor +RUN yum clean all +COPY nifisupervisord.conf /etc/supervisord.conf +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] diff --git a/roles/docker/tasks/nifi.yml b/roles/docker/tasks/nifi.yml index 7023444e9c0181dcde646dd0f1b4e55dc1f2b031..e85fb61a10993c070180291d6c1ff8d06c74dece 100644 --- a/roles/docker/tasks/nifi.yml +++ b/roles/docker/tasks/nifi.yml @@ -10,7 +10,6 @@ networks_cli_compatible: yes volumes: - "{{item}}:/opt/nifi/nifi-current/conf" - entrypoint: "/bin/bash" interactive: "yes" with_items: "{{ groups['nificontainers'] }}" tags: diff --git a/roles/nifi/tasks/main.yml b/roles/nifi/tasks/main.yml index 790bf4a030c33fbbd43e6c5d34df9ea0ded84914..392a6dd38028abd302277e5093e3bdbb8a6d4567 100644 --- a/roles/nifi/tasks/main.yml +++ b/roles/nifi/tasks/main.yml @@ -15,6 +15,7 @@ - start - name: Copy certificates in NiFi conf dir + remote_user: nifi copy: src: "{{ item }}" dest: "conf/{{ item }}" @@ -25,6 +26,7 @@ - start - name: Configure flow.xml + remote_user: nifi template: src: "flow.xml.j2" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" @@ -32,6 +34,7 @@ - start - name: Gzip flow.xml + remote_user: nifi archive: path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" @@ -40,12 +43,14 @@ - start - name: Get openid authkey + remote_user: nifi set_fact: nifisecret: "{{lookup('file', 'files/nifisecret',convert_data=False) | from_json }}" tags: - start - name: Configure NiFi boostrap properties + remote_user: nifi template: src: bootstrap.conf.j2 dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf" @@ -53,6 +58,7 @@ - start - name: Configure NiFi properties for secure servers + remote_user: nifi template: src: nifi.properties.j2 dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties" @@ -60,6 +66,7 @@ - start - name: Copy authorizations.xml + remote_user: nifi copy: src: "authorizations.xml" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml" @@ -67,6 +74,7 @@ - start - name: Configure users + remote_user: nifi template: src: users.xml.j2 dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml" @@ -74,6 +82,7 @@ - start - name: Configure NiFi authorizers for secure servers + remote_user: nifi template: src: authorizers.xml.j2 dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml" @@ -81,16 +90,19 @@ - start - name: Create conf/enrich dir + remote_user: nifi file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory tags: - start - name: Create conf/enrich/freq dir + remote_user: nifi file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory tags: - start - name: Download freq processor + remote_user: nifi get_url: url: 'https://scm.uninett.no/geant-wp8-t3.1/nifi-processors/-/raw/master/scripts/freq/{{ item }}' dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/" @@ -102,6 +114,7 @@ - start - name: Copy empty GeoLite2-City database + remote_user: nifi copy: src: GeoLite2-City.mmdb dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb" @@ -109,6 +122,7 @@ - start - name: Copy CountriesWithRegionalCodes.csv + remote_user: nifi copy: src: CountriesWithRegionalCodes.csv dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv" @@ -116,6 +130,7 @@ - start - name: Create empty list of Tor nodes + remote_user: nifi copy: content: "ip_addr,value" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv" @@ -124,6 +139,7 @@ - start - name: Download umbrella-top-1m.csv.zip + remote_user: nifi local_action: module: get_url url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip @@ -133,6 +149,7 @@ - start - name: Unzip umbrella-top-1m.csv.zip + remote_user: nifi local_action: module: unarchive src: "/tmp/umbrella-top-1m.csv.zip" @@ -142,6 +159,7 @@ - start - name: Copy umbrella-top-1m.csv + remote_user: nifi copy: src: "/tmp/top-1m.csv" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" @@ -149,6 +167,7 @@ - start - name: Add header to umbrella-top-1m.csv + remote_user: nifi lineinfile: path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" line: 'index,domain' @@ -157,6 +176,7 @@ - start - name: Download alexa-top-1m.csv.zip + remote_user: nifi local_action: module: get_url url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip @@ -166,6 +186,7 @@ - start - name: Unzip alexa-top-1m.csv.zip + remote_user: nifi local_action: module: unarchive src: "/tmp/alexa-top-1m.csv.zip" @@ -175,6 +196,7 @@ - start - name: Copy alexa-top-1m.csv + remote_user: nifi copy: src: "/tmp/top-1m.csv" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" @@ -182,6 +204,7 @@ - start - name: Add header to alexa-top-1m.csv + remote_user: nifi lineinfile: path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" line: 'index,domain' @@ -190,7 +213,8 @@ - start - name: Start NiFi - command: "{{ ansible_facts.env['NIFI_HOME'] }}/bin/nifi.sh start" + remote_user: root + command: "supervisorctl start nifi" tags: - start @@ -204,8 +228,14 @@ # - group_by: key=reachable # when: ping_result|success +- name: Set Autostart for supervisord's services + shell: "sed -i 's/autostart=false/autostart=true/g' /etc/supervisord.conf" + tags: + - start + - name: Stop NiFi - command: "{{ ansible_facts.env['NIFI_HOME'] }}/bin/nifi.sh stop" + remote_user: root + command: "supervisorctl stop nifi" tags: - stop