diff --git a/inventories/filebeat b/inventories/filebeat
index e8df75e149c553272de4cf36dfcb71742f94c2e5..f4600cf1d042698cdc4fd5e4d4edba1e81b00264 100644
--- a/inventories/filebeat
+++ b/inventories/filebeat
@@ -5,7 +5,7 @@ soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
#soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="text"
#soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text"
-soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text"
+soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="text"
soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2
index 8e8cc20319c706a462f7b2b4719e4e977c2b2cac..3956cebc1cdfe2db106af5a70edb416b0277ef26 100644
--- a/roles/nifi/templates/flow.xml.j2
+++ b/roles/nifi/templates/flow.xml.j2
@@ -4226,16 +4226,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
- <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
- <name>To data output</name>
- <position x="-632.0" y="328.0" />
+ <id>27d5761b-0172-1000-0000-000059275dad</id>
+ <name>To enrichment</name>
+ <position x="-312.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
<outputPort>
- <id>27d5761b-0172-1000-0000-000059275dad</id>
- <name>To enrichment</name>
- <position x="-312.0" y="328.0" />
+ <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
+ <name>To data output</name>
+ <position x="-632.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
@@ -4273,6 +4273,10 @@
<name>Routing Strategy</name>
<value>Route to Property name</value>
</property>
+ <property>
+ <name>kibana</name>
+ <value>${log_type:equals("kibana")}</value>
+ </property>
<property>
<name>suricata</name>
<value>${log_type:equals("suricata")}</value>
@@ -5100,14 +5104,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>895faa7a-0175-1000-0000-000014ef9dd3</id>
- <position x="278.84829417593915" y="332.4492766741185" />
- </funnel>
<funnel>
<id>895f7db3-0175-1000-ffff-ffff8229d688</id>
<position x="-1446.1517058240609" y="301.4492766741185" />
</funnel>
+ <funnel>
+ <id>895faa7a-0175-1000-0000-000014ef9dd3</id>
+ <position x="278.84829417593915" y="332.4492766741185" />
+ </funnel>
<connection>
<id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
<name />
@@ -6732,6 +6736,10 @@
<name>data_index</name>
<value>logs-haproxy</value>
</property>
+ <property>
+ <name>enrich_ip1</name>
+ <value>/client.ip</value>
+ </property>
</processor>
<inputPort>
<id>65a33e05-e157-1bfc-8741-adf11b3df720</id>
@@ -6747,14 +6755,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id>
- <position x="1882.9999517774115" y="327.9999931568573" />
- </funnel>
<funnel>
<id>312d3490-461e-13ac-a3a2-603704c456e2</id>
<position x="8.0" y="424.0" />
</funnel>
+ <funnel>
+ <id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id>
+ <position x="1882.9999517774115" y="327.9999931568573" />
+ </funnel>
<connection>
<id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id>
<name />
@@ -7132,14 +7140,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
- <position x="248.5321508445502" y="703.4412774751572" />
- </funnel>
<funnel>
<id>06521038-335b-3139-839d-ab43a013ce03</id>
<position x="-1557.869726298236" y="758.8984861527665" />
</funnel>
+ <funnel>
+ <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
+ <position x="248.5321508445502" y="703.4412774751572" />
+ </funnel>
<connection>
<id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
<name />
@@ -7261,6 +7269,169 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
+ <processGroup>
+ <id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
+ <name>Kibana</name>
+ <position x="-440.0" y="864.0" />
+ <comment />
+ <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
+ <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
+ <processor>
+ <id>992c3710-1c87-169c-ab17-d2597387a25e</id>
+ <name>UpdateAttribute</name>
+ <position x="360.0" y="512.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-kibana</value>
+ </property>
+ </processor>
+ <inputPort>
+ <id>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</id>
+ <name>Input</name>
+ <position x="408.0" y="320.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </inputPort>
+ <outputPort>
+ <id>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</id>
+ <name>Output</name>
+ <position x="408.0" y="760.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </outputPort>
+ <connection>
+ <id>cc403fb4-8d68-1c68-82c3-b9af4affddaa</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</sourceId>
+ <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>992c3710-1c87-169c-ab17-d2597387a25e</destinationId>
+ <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>b9e33c29-910f-134a-8390-2970800d7fcf</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>992c3710-1c87-169c-ab17-d2597387a25e</sourceId>
+ <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</destinationId>
+ <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <connection>
+ <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+ <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>349b339b-a821-1197-0000-00002e648df6</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+ <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+ <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>89639d3d-0175-1000-ffff-ffffb446c257</destinationId>
+ <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship>unmatched</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>8d1fe825-0175-1000-ffff-fffff0505cdc</id>
<name />
@@ -7301,6 +7472,26 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>61c51cd8-0176-1000-ffff-ffff9247ba7c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</sourceId>
+ <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+ <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>bc6e50cc-0175-1000-ffff-ffffbd982e0c</id>
<name />
@@ -7344,18 +7535,20 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
+ <id>6196cd03-0176-1000-ffff-ffffd39b8c82</id>
<name />
- <bendPoints />
- <labelIndex>1</labelIndex>
+ <bendPoints>
+ <bendPoint x="-576.0" y="896.0" />
+ </bendPoints>
+ <labelIndex>0</labelIndex>
<zIndex>0</zIndex>
- <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
+ <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+ <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</destinationId>
+ <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship>kibana</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -7384,13 +7577,13 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>349b339b-a821-1197-0000-00002e648df6</id>
+ <id>349b3303-a821-1197-ffff-ffffa12b866d</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</sourceId>
- <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceId>8963b202-0175-1000-0000-000022d64ba2</sourceId>
+ <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
<destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
@@ -7423,46 +7616,6 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
- <connection>
- <id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>89639d3d-0175-1000-ffff-ffffb446c257</destinationId>
- <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>unmatched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b3303-a821-1197-ffff-ffffa12b866d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8963b202-0175-1000-0000-000022d64ba2</sourceId>
- <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
<connection>
<id>349b3301-a821-1197-0000-0000070259c4</id>
<name />
@@ -7996,7 +8149,7 @@
</property>
<property>
<name>Password</name>
- <value>enc{1c9a67efa861b9a5f0ced47e1bb930650b19b788b8576e55d87fa2a3a4760d790d7425f299ed70ea1859a64a26753959}</value>
+ <value>enc{2d7036ed427615cc0da2c105923da69609e9a5b2cfdf3ae7356c2fb11de6538a5393d363e717b6316763851a10ca5679}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
@@ -11008,7 +11161,7 @@
</property>
<property>
<name>Truststore Password</name>
- <value>enc{d064a1e3a5a974d37b0202bbb9551137b9543af176d965ad630f0fc2bdafa690}</value>
+ <value>enc{f1a53d9f8ccdcff528b762ffc26710276eb38abb97f6abe2fd3fb2e8779ca390}</value>
</property>
<property>
<name>Truststore Type</name>