Merge branch 'master' of https://gitlab.geant.org/gn4-3-wp8-t3.1-soc/soctools into new_names

inventories/filebeat

@@ -3,9 +3,9 @@ soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
 soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
-#soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="json"
-#soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="json"
-soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text"
+#soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="text"
+#soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text"
+soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text"
 soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="text"
 soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
 soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"

roles/build/files/cassandrasupervisord.conf

@@ -30,3 +30,15 @@ stderr_logfile_backups = 0
 stderr_logfile = /var/log/supervisor/cassandra_stderr.log
 stdout_logfile = /var/log/supervisor/cassandra_stdout.log
 
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/files/cortexsupervisord.conf

@@ -41,3 +41,16 @@ stdout_logfile_backups = 0
 stderr_logfile_backups = 0
 stderr_logfile = /var/log/supervisor/cortex_stderr.log
 stdout_logfile = /var/log/supervisor/cortex_stdout.log
+
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/files/keycloaksupervisord.conf

@@ -30,3 +30,15 @@ stderr_logfile_backups = 0
 stderr_logfile = /var/log/supervisor/keycloak_stderr.log
 stdout_logfile = /var/log/supervisor/keycloak_stdout.log
 
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/files/kibanasupervisord.conf

@@ -18,16 +18,26 @@ serverurl=unix:///tmp/supervisor.sock
 
 [program:kibana]
 user=kibana
+group=kibana
 directory=/usr/share/kibana
-command=sh -c "/usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml > kblog 2>&1"
-
-[program:cron]
-autostart=true
+command=sh -c "/usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml"
+autostart=false
 autorestart=true
-command=crond -n
 logfile_maxbytes=10MB
 stdout_logfile_backups = 0
 stderr_logfile_backups = 0
-stderr_logfile = /var/log/supervisor/cron_stderr.log
-stdout_logfile = /var/log/supervisor/cron_stdout.log
+stderr_logfile = /var/log/supervisor/kibana_stderr.log
+stdout_logfile = /var/log/supervisor/kibana_stdout.log
 
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/files/odfesupervisord.conf

+[unix_http_server]
+file=/tmp/supervisor.sock
+
+[supervisord]
+pidfile=/tmp/supervisord.pid
+nodaemon=true
+logfile=/var/log/supervisor/supervisord.log
+logfile_maxbytes=10MB
+logfile_backups=10
+loglevel=info
+childlogdir=/var/log/supervisor/
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///tmp/supervisor.sock
+
+[program:odfe]
+user=elasticsearch
+group=elasticsearch
+directory=/usr/share/elasticsearch
+command=sh -c "/usr/share/elasticsearch/bin/elasticsearch"
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/elasticsearch_stderr.log
+stdout_logfile = /var/log/supervisor/elasticsearch_stdout.log
+
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/files/thehivesupervisord.conf

@@ -28,3 +28,16 @@ stdout_logfile_backups = 0
 stderr_logfile_backups = 0
 stderr_logfile = /var/log/supervisor/thehive_stderr.log
 stdout_logfile = /var/log/supervisor/thehive_stdout.log
+
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/files/zookeepersupervisord.conf

@@ -28,3 +28,16 @@ stdout_logfile_backups = 0
 stderr_logfile_backups = 0
 stderr_logfile = /var/log/supervisor/zookeeper_stderr.log
 stdout_logfile = /var/log/supervisor/zookeeper_stdout.log
+
+[program:filebeat]
+directory=/opt/filebeat
+user=root
+group=root
+command=/bin/bash -c '/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml'
+autostart=false
+autorestart=true
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/filebeat_stderr.log
+stdout_logfile = /var/log/supervisor/filebeat_stdout.log

roles/build/templates/odfees/Dockerfile-elastic.j2

@@ -18,6 +18,5 @@ RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
 RUN echo 'elasticsearch ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
 
 EXPOSE 9200 9300
-USER elasticsearch
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
 

roles/build/templates/odfees/Dockerfile-odfeelastic.j2

@@ -13,6 +13,7 @@ RUN for PLUGIN in \
     chown -R elasticsearch plugins/opendistro_security
 
 RUN echo 'elasticsearch ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-USER elasticsearch
-
+RUN yum install -y supervisor
+RUN yum clean all
+COPY odfesupervisord.conf /etc/supervisord.conf
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

roles/build/templates/odfekibana/Dockerfile-kibana.j2

@@ -16,10 +16,8 @@ RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
     chown -R kibana /usr/share/kibana/config/
 
 RUN echo 'kibana ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-COPY kibanasupervisord.conf /etc/supervisord.conf
-COPY kibanalogrotate.conf /etc/logrotate.d/kibana.conf
 
 EXPOSE 5601
-USER kibana
-ENTRYPOINT ["/bin/bash"]
+COPY kibanasupervisord.conf /etc/supervisord.conf
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
 

roles/build/templates/odfekibana/Dockerfile-odfekibana.j2

@@ -14,5 +14,4 @@ RUN for PLUGIN in \
 ADD thehive_button /usr/share/kibana/plugins/thehive_button
 RUN chown -R kibana:kibana /usr/share/kibana/plugins/thehive_button
 
-RUN echo 'kibana ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

roles/docker/tasks/odfees.yml

@@ -10,7 +10,6 @@
     networks_cli_compatible: yes
     volumes:
       - "{{item}}:/usr/share/elasticsearch/data"
-    entrypoint: "/bin/bash"
     interactive: "yes"
   with_items: "{{ groups['odfeescontainers'] }}"
   tags:

roles/docker/tasks/odfekibana.yml

@@ -10,7 +10,6 @@
     networks_cli_compatible: yes
     published_ports:
       - "5601:5601"
-    entrypoint: "/bin/bash"
     interactive: "yes"
   with_items: "{{ groups['odfekibanacontainers'] }}"
   tags:

roles/odfees/tasks/main.yml

@@ -15,6 +15,7 @@
     - start
 
 - name: Copy certificates in odfe conf dir
+  remote_user: elasticsearch
   copy:
     src:  "{{ item }}"
     dest: "config/{{ item }}"
@@ -27,6 +28,7 @@
     - start
 
 - name: Configure sysconfig
+  remote_user: elasticsearch
   template:
     src: sysconfig_elasticsearch.j2
     dest: sysconfig_elasticsearch
@@ -34,11 +36,13 @@
     - start
 
 - name: Copy sysconfig to /etc
+  remote_user: elasticsearch
   command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
   tags: 
     - start
 
 - name: Configure odfe properties
+  remote_user: elasticsearch
   template:
     src: "config/{{item}}.j2"
     dest: "config/{{item}}"
@@ -50,6 +54,7 @@
     - start
 
 - name: Change password for admin
+  remote_user: elasticsearch
   command: "bash plugins/opendistro_security/tools/hash.sh -p {{odfees_adminpass}}"
   register: adminhash
   # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
@@ -59,10 +64,12 @@
 - set_fact:
     adminhashpwd: "{{ adminhash.stdout }}"
     #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}"
+  remote_user: elasticsearch
   tags:
     - start
 
 - name: Change password for cortex
+  remote_user: elasticsearch
   command: "bash plugins/opendistro_security/tools/hash.sh -p {{cortex_odfe_pass}}"
   register: cortexhash
   # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
@@ -72,10 +79,12 @@
 - set_fact:
     cortexhashpwd: "{{ cortexhash.stdout }}"
     #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}"
+  remote_user: elasticsearch
   tags:
     - start
 
 - name: Configure opendistro_security properties
+  remote_user: elasticsearch
   template:
     src: "securityconfig/{{item}}.j2"
     dest: "plugins/opendistro_security/securityconfig/{{item}}"
@@ -92,11 +101,13 @@
 #    - start
 
 - name: Start OpenDistro for Elasticsearch
-  command: "/usr/share/elasticsearch/bin/elasticsearch -p {{ inventory_hostname }}.pid -d"
+  remote_user: root
+  command: "supervisorctl start odfe"
   tags:
     - start
 
 - name: Wait for ElasticSearch
+  remote_user: root
   wait_for:
     host: "{{groups['odfeescontainers'][0]}}"
     port: 9200
@@ -106,13 +117,21 @@
     - start
 
 - name: Configure OpenDistro security
+  remote_user: elasticsearch
   command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{soctools_users[0].CN}}.p12' -kspass {{soctools_users[0].password}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{tspass}} -cn soctools-cluster"
   when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
   tags:
     - start
 
+- name: Set Autostart for supervisord's services
+  remote_user: root
+  shell: "sed -i 's/autostart=false/autostart=true/g' /etc/supervisord.conf"
+  tags:
+    - start
+
 - name: Stop OpenDistro for Elasticsearch
-  command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+  remote_user: root
+  command: "supervisorctl stop odfe"
   tags:
     - stop
 

roles/odfekibana/tasks/main.yml

@@ -101,9 +101,9 @@
     - start
 
 
-- name: Start Supervisord (kibana and cron)
+- name: Start Kibana
   remote_user: root
-  shell: "/usr/bin/supervisord -c /etc/supervisord.conf &"
+  shell: "supervisorctl start kibana"
   tags:
     - start
 
@@ -203,8 +203,13 @@
 #  tags:
 #    - stop
 
+- name: Set Autostart for supervisord's services
+  shell: "sed -i 's/autostart=false/autostart=true/g' /etc/supervisord.conf"
+  tags:
+    - start
+
 - name: Stop OpenDistro Kibana for Elasticsearch
   remote_user: root
-  command: "pkill -SIGTERM -F /tmp/supervisord.pid"
+  command: "supervisorctl stop kibana"
   tags:
     - stop