diff --git a/roles/filebeat/templates/filebeat.yml.j2 b/roles/filebeat/templates/filebeat.yml.j2 index 3335c48b7d3d00b1b9ff8733ed22007be776594d..6fb7351a22fcf3928116a0f6ee998f938b633031 100644 --- a/roles/filebeat/templates/filebeat.yml.j2 +++ b/roles/filebeat/templates/filebeat.yml.j2 @@ -5,6 +5,11 @@ filebeat.inputs: {% for file in FILEBEAT_FILES %} - {{ file }} {% endfor %} +{% if FILEBEAT_LOG_FORMAT == 'json' %} + json.keys_under_root: true + json.overwrite_keys: true + json.add_error_key: true +{% endif %} {% else %} - type: syslog protocol.udp: diff --git a/roles/odfees/templates/config/log4j2.properties.j2 b/roles/odfees/templates/config/log4j2.properties.j2 index 9ad290ad82679309319cee88bee3eaf9d49814eb..ee01d9a1406720d46fe983efacf16cc8d52c3729 100644 --- a/roles/odfees/templates/config/log4j2.properties.j2 +++ b/roles/odfees/templates/config/log4j2.properties.j2 @@ -5,5 +5,27 @@ appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 20MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 100MB + rootLogger.level = info -rootLogger.appenderRef.console.ref = console +#rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling diff --git a/soctools-inventory b/soctools-inventory index f8897eb15739f01229c97f817908f7853db84e44..616f34a1b736d9ee66d8e208b05fde7175155a04 100644 --- a/soctools-inventory +++ b/soctools-inventory @@ -35,14 +35,14 @@ dsoclab-cortex ansible_connection=docker dsoclab-haproxy ansible_connection=docker [filebeat] -dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" -dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" -dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" -#dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='[""]' -#dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='[""]' -dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana" +dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" +dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/dsoclab-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="json" +dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/dsoclab-cluster_server.json"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="json" +dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='["/usr/share/kibana/kblog"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text" #dsoclab-keycloak ansible_connection=docker FILEBEAT_FILES='[""]' -dsoclab-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" +dsoclab-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text" #dsoclab-misp ansible_connection=docker FILEBEAT_FILES='[""]' -dsoclab-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" +dsoclab-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text" #dsoclab-zookeeper ansible_connection=docker FILEBEAT_FILES='[""]'