diff --git a/roles/build/files/misp_rh-php72-php-fpm b/roles/build/files/misp_rh-php72-php-fpm new file mode 100644 index 0000000000000000000000000000000000000000..2d67a51c4a2c69132897823a698bef39f4ee0fa6 --- /dev/null +++ b/roles/build/files/misp_rh-php72-php-fpm @@ -0,0 +1,21 @@ +/var/opt/rh/rh-php72/log/php-fpm/error.log { + missingok + notifempty + sharedscripts + delaycompress + postrotate + /bin/kill -SIGUSR1 `cat /var/opt/rh/rh-php72/run/php-fpm/php-fpm.pid 2>/dev/null` 2>/dev/null || true + endscript +} + +/var/opt/rh/rh-php72/log/php-fpm/www-*log { + su apache apache + create 600 apache apache + missingok + notifempty + sharedscripts + delaycompress + postrotate + /bin/kill -SIGUSR1 `cat /var/opt/rh/rh-php72/run/php-fpm/php-fpm.pid 2>/dev/null` 2>/dev/null || true + endscript +} diff --git a/roles/build/templates/misp/Dockerfile.j2 b/roles/build/templates/misp/Dockerfile.j2 index b99d2a84cffc07c99a2622c5bba30c3e479a7425..3b2ce45c11a85cdc8e31f421468db70a099d4cac 100644 --- a/roles/build/templates/misp/Dockerfile.j2 +++ b/roles/build/templates/misp/Dockerfile.j2 @@ -76,9 +76,11 @@ RUN chown -R apache:apache /var/www/MISP ; \ chmod -R g+ws /var/www/MISP/app/files ; \ chmod -R g+ws /var/www/MISP/app/files/scripts/tmp +COPY misp_rh-php72-php-fpm /etc/logrotate.d/rh-php72-php-fpm + # 80/443 - MISP web server, 3306 - mysql, 6379 - redis, 6666 - MISP modules, 50000 - MISP ZeroMQ EXPOSE 80 443 6443 6379 6666 50000 COPY mispsupervisord.conf /etc/supervisord.conf -#ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] diff --git a/roles/build/templates/misp/supervisord.conf.j2 b/roles/build/templates/misp/supervisord.conf.j2 index fbd4dea3bbd292aa2eea51772ecc559dc0b51703..ee1e0c4121b352d854d1e26f7bb67d917688ce85 100644 --- a/roles/build/templates/misp/supervisord.conf.j2 +++ b/roles/build/templates/misp/supervisord.conf.j2 @@ -1,25 +1,70 @@ +[unix_http_server] +file=/tmp/supervisor.sock + [supervisord] -nodaemon=false +pidfile=/tmp/supervisord.pid +nodaemon=true +logfile=/var/log/supervisor/supervisord.log +logfile_maxbytes=10MB +logfile_backups=10 +loglevel=info +childlogdir=/var/log/supervisor/ user=root +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock + +[program:cron] +autostart=true +autorestart=true +command=crond -n +logfile_maxbytes=10MB +stdout_logfile_backups = 0 +stderr_logfile_backups = 0 +stderr_logfile = /var/log/supervisor/cron_stderr.log +stdout_logfile = /var/log/supervisor/cron_stdout.log + [program:php-fpm] # EnvironmentFile=/etc/opt/rh/rh-php72/sysconfig/php-fpm command=/opt/rh/rh-php72/root/usr/sbin/php-fpm --nodaemonize +autostart=false +autorestart=true [program:redis-server] process_name = redis-server directory = /var/opt/rh/rh-redis32/lib/redis/ command=/opt/rh/rh-redis32/root/usr/bin/redis-server /etc/opt/rh/rh-redis32/redis.conf user=redis +autostart=false +autorestart=true [program:apache2] command=/usr/sbin/httpd -DFOREGROUND +autostart=false +autorestart=true [program:misp-modules] command=/bin/bash -c "/usr/local/bin/misp-modules -l '0.0.0.0' -s" user = apache +autostart=false +autorestart=unexpected startsecs = 0 +logfile_maxbytes=10MB +stdout_logfile_backups = 0 +stderr_logfile_backups = 0 +stderr_logfile = /var/log/supervisor/misp-modules_stderr.log +stdout_logfile = /var/log/supervisor/misp-modules_stdout.log [program:workers] command=/bin/bash /var/www/MISP/app/Console/worker/start.sh user=apache +autostart=false +autorestart=unexpected +logfile_maxbytes=10MB +stdout_logfile_backups = 0 +stderr_logfile_backups = 0 +stderr_logfile = /var/log/supervisor/workers_stderr.log +stdout_logfile = /var/log/supervisor/workers_stdout.log diff --git a/roles/docker/tasks/misp.yml b/roles/docker/tasks/misp.yml index 34a42ad3c033abf3c58a1664a92205b9cb76f69e..18520a14abe069a2b76040f3fab60932818fd60a 100644 --- a/roles/docker/tasks/misp.yml +++ b/roles/docker/tasks/misp.yml @@ -8,8 +8,6 @@ networks: - name: "{{ soctools_netname}}" networks_cli_compatible: yes - entrypoint: "/bin/bash" - interactive: "yes" published_ports: - "6443:6443" tags: diff --git a/roles/filebeat/templates/filebeat.yml.j2 b/roles/filebeat/templates/filebeat.yml.j2 index 3335c48b7d3d00b1b9ff8733ed22007be776594d..6fb7351a22fcf3928116a0f6ee998f938b633031 100644 --- a/roles/filebeat/templates/filebeat.yml.j2 +++ b/roles/filebeat/templates/filebeat.yml.j2 @@ -5,6 +5,11 @@ filebeat.inputs: {% for file in FILEBEAT_FILES %} - {{ file }} {% endfor %} +{% if FILEBEAT_LOG_FORMAT == 'json' %} + json.keys_under_root: true + json.overwrite_keys: true + json.add_error_key: true +{% endif %} {% else %} - type: syslog protocol.udp: diff --git a/roles/misp/tasks/main.yml b/roles/misp/tasks/main.yml index 826e9dce14466e41450089a3720f28d4a4e54573..773273d4f63e05489213c2328df51fbee960b85b 100644 --- a/roles/misp/tasks/main.yml +++ b/roles/misp/tasks/main.yml @@ -52,8 +52,8 @@ tags: - start -- name: Start MISP - command: "/usr/bin/supervisord -c /etc/supervisord.conf" +- name: Start MISP Services + command: "supervisorctl start all" tags: - start diff --git a/roles/odfees/templates/config/log4j2.properties.j2 b/roles/odfees/templates/config/log4j2.properties.j2 index 9ad290ad82679309319cee88bee3eaf9d49814eb..ee01d9a1406720d46fe983efacf16cc8d52c3729 100644 --- a/roles/odfees/templates/config/log4j2.properties.j2 +++ b/roles/odfees/templates/config/log4j2.properties.j2 @@ -5,5 +5,27 @@ appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 20MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 100MB + rootLogger.level = info -rootLogger.appenderRef.console.ref = console +#rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling diff --git a/soctools-inventory b/soctools-inventory index f8897eb15739f01229c97f817908f7853db84e44..d80ce10ed2fa9fa3719ab6741529758ed514a7c1 100644 --- a/soctools-inventory +++ b/soctools-inventory @@ -35,14 +35,14 @@ dsoclab-cortex ansible_connection=docker dsoclab-haproxy ansible_connection=docker [filebeat] -dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" -dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" -dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" -#dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='[""]' -#dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='[""]' -dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana" +dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/dsoclab-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="json" +dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/dsoclab-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="json" +dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text" #dsoclab-keycloak ansible_connection=docker FILEBEAT_FILES='[""]' -dsoclab-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" -#dsoclab-misp ansible_connection=docker FILEBEAT_FILES='[""]' -dsoclab-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" +dsoclab-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text" +dsoclab-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text" +dsoclab-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text" #dsoclab-zookeeper ansible_connection=docker FILEBEAT_FILES='[""]'