Merge branch 'master' of gitlab.geant.org:gn4-3-wp8-t3.1-soc/soctools

roles/build/files/misp_rh-php72-php-fpm

+/var/opt/rh/rh-php72/log/php-fpm/error.log {
+    missingok
+    notifempty
+    sharedscripts
+    delaycompress
+    postrotate
+        /bin/kill -SIGUSR1 `cat /var/opt/rh/rh-php72/run/php-fpm/php-fpm.pid 2>/dev/null` 2>/dev/null || true
+    endscript
+}
+
+/var/opt/rh/rh-php72/log/php-fpm/www-*log {
+    su apache apache
+    create 600 apache apache
+    missingok
+    notifempty
+    sharedscripts
+    delaycompress
+    postrotate
+        /bin/kill -SIGUSR1 `cat /var/opt/rh/rh-php72/run/php-fpm/php-fpm.pid 2>/dev/null` 2>/dev/null || true
+    endscript
+}

roles/build/templates/misp/Dockerfile.j2

@@ -76,9 +76,11 @@ RUN chown -R apache:apache /var/www/MISP ; \
     chmod -R g+ws /var/www/MISP/app/files ; \
     chmod -R g+ws /var/www/MISP/app/files/scripts/tmp
 
+COPY misp_rh-php72-php-fpm /etc/logrotate.d/rh-php72-php-fpm
+
 # 80/443 - MISP web server, 3306 - mysql, 6379 - redis, 6666 - MISP modules, 50000 - MISP ZeroMQ
 EXPOSE 80 443 6443 6379 6666 50000
 
 COPY mispsupervisord.conf /etc/supervisord.conf
-#ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
 

roles/build/templates/misp/supervisord.conf.j2

+[unix_http_server]
+file=/tmp/supervisor.sock
+
 [supervisord]
-nodaemon=false
+pidfile=/tmp/supervisord.pid
+nodaemon=true
+logfile=/var/log/supervisor/supervisord.log
+logfile_maxbytes=10MB
+logfile_backups=10
+loglevel=info
+childlogdir=/var/log/supervisor/
 user=root
 
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///tmp/supervisor.sock
+
+[program:cron]
+autostart=true
+autorestart=true
+command=crond -n
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/cron_stderr.log
+stdout_logfile = /var/log/supervisor/cron_stdout.log
+
 [program:php-fpm]
 # EnvironmentFile=/etc/opt/rh/rh-php72/sysconfig/php-fpm
 command=/opt/rh/rh-php72/root/usr/sbin/php-fpm --nodaemonize
+autostart=false
+autorestart=true
 
 [program:redis-server]
 process_name = redis-server
 directory = /var/opt/rh/rh-redis32/lib/redis/
 command=/opt/rh/rh-redis32/root/usr/bin/redis-server /etc/opt/rh/rh-redis32/redis.conf
 user=redis
+autostart=false
+autorestart=true
 
 [program:apache2]
 command=/usr/sbin/httpd -DFOREGROUND
+autostart=false
+autorestart=true
 
 [program:misp-modules]
 command=/bin/bash -c "/usr/local/bin/misp-modules -l '0.0.0.0' -s"
 user = apache
+autostart=false
+autorestart=unexpected
 startsecs = 0
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/misp-modules_stderr.log
+stdout_logfile = /var/log/supervisor/misp-modules_stdout.log
 
 [program:workers]
 command=/bin/bash /var/www/MISP/app/Console/worker/start.sh
 user=apache
+autostart=false
+autorestart=unexpected
+logfile_maxbytes=10MB
+stdout_logfile_backups = 0
+stderr_logfile_backups = 0
+stderr_logfile = /var/log/supervisor/workers_stderr.log
+stdout_logfile = /var/log/supervisor/workers_stdout.log

roles/docker/tasks/misp.yml

@@ -8,8 +8,6 @@
     networks:
       - name: "{{ soctools_netname}}"
     networks_cli_compatible: yes
-    entrypoint: "/bin/bash"
-    interactive: "yes"
     published_ports:
       - "6443:6443"
   tags:

roles/filebeat/templates/filebeat.yml.j2

@@ -5,6 +5,11 @@ filebeat.inputs:
 {% for file in FILEBEAT_FILES %}
       - {{ file }}
 {% endfor %}
+{% if FILEBEAT_LOG_FORMAT == 'json' %}
+  json.keys_under_root: true
+  json.overwrite_keys: true
+  json.add_error_key: true
+{% endif %}
 {% else %}
 - type: syslog
   protocol.udp:

roles/misp/tasks/main.yml

@@ -52,8 +52,8 @@
   tags:
     - start
 
-- name: Start MISP
-  command: "/usr/bin/supervisord -c /etc/supervisord.conf"
+- name: Start MISP Services
+  command: "supervisorctl start all"
   tags:
     - start
 

roles/odfees/templates/config/log4j2.properties.j2

@@ -5,5 +5,27 @@ appender.console.name = console
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
 
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json
+appender.rolling.layout.type = ESJsonLayout
+appender.rolling.layout.type_name = server
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 20MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.fileIndex = nomax
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
+appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
+appender.rolling.strategy.action.condition.nested_condition.exceeds = 100MB
+
 rootLogger.level = info
-rootLogger.appenderRef.console.ref = console
+#rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling

soctools-inventory

@@ -35,14 +35,14 @@ dsoclab-cortex ansible_connection=docker
 dsoclab-haproxy ansible_connection=docker
 
 [filebeat]
-dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi"
-dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi"
-dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi"
-#dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='[""]'
-#dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='[""]'
-dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana"
+dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
+dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
+dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
+dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/dsoclab-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="json"
+dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/dsoclab-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="json"
+dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text"
 #dsoclab-keycloak ansible_connection=docker FILEBEAT_FILES='[""]'
-dsoclab-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql"
-#dsoclab-misp ansible_connection=docker FILEBEAT_FILES='[""]'
-dsoclab-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy"
+dsoclab-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
+dsoclab-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
+dsoclab-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
 #dsoclab-zookeeper ansible_connection=docker FILEBEAT_FILES='[""]'