diff --git a/inventories/filebeat b/inventories/filebeat
index f4600cf1d042698cdc4fd5e4d4edba1e81b00264..cf6fb49060191fecf1d22704c367c9499929dd53 100644
--- a/inventories/filebeat
+++ b/inventories/filebeat
@@ -6,7 +6,7 @@ soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/l
#soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="text"
#soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text"
soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
-soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="text"
+soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="zookeeper" FILEBEAT_LOG_FORMAT="text"
diff --git a/roles/keycloak/tasks/main.yml b/roles/keycloak/tasks/main.yml
index 296d051e9a316d7049613c6b4b6c9b084d47ebf1..41a832c1992542b6cdf480d2eb1b8a996c6c067a 100644
--- a/roles/keycloak/tasks/main.yml
+++ b/roles/keycloak/tasks/main.yml
@@ -36,6 +36,33 @@
tags:
- start
+- name: Configure logging format
+ remote_user: jboss
+ lineinfile: #TODO: Change to community.general.xml
+ path: /opt/jboss/keycloak/standalone/configuration/standalone.xml
+ regexp: '.*<formatter name="PATTERN">.*'
+ line: "<formatter name=\"JSON\"><json-formatter date-format=\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\" pretty-print=\"false\" print-details=\"true\" zone-id=\"UTC\"/></formatter><formatter name=\"PATTERN\">"
+ tags:
+ - start
+
+- name: Enable event logging
+ remote_user: jboss
+ lineinfile:
+ path: /opt/jboss/keycloak/standalone/configuration/standalone.xml
+ regexp: '.*<spi name="eventsStore">.*'
+ line: '<spi name="eventsListener"><provider name="jboss-logging" enabled="true"><properties><property name="success-level" value="INFO"/><property name="error-level" value="WARN"/></properties></provider></spi><spi name="eventsStore">'
+ tags:
+ - start
+
+- name: Specify logging format
+ remote_user: jboss
+ lineinfile:
+ path: /opt/jboss/keycloak/standalone/configuration/standalone.xml
+ regexp: ".*<named-formatter name=.*"
+ line: '<named-formatter name="JSON"/>'
+ tags:
+ - start
+
- name: Configure Keycloak start script
remote_user: jboss
template:
@@ -48,7 +75,6 @@
tags:
- start
-
- name: Start Keycloak IdP
remote_user: root
command: "supervisorctl start keycloak"
diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2
index 3956cebc1cdfe2db106af5a70edb416b0277ef26..86f3a35d81acd93dfa56891efbe9c5b791a7b79e 100644
--- a/roles/nifi/templates/flow.xml.j2
+++ b/roles/nifi/templates/flow.xml.j2
@@ -4205,16 +4205,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
- <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
- <name>To data output</name>
- <position x="-1120.0" y="592.0" />
+ <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
+ <name>To enrichment</name>
+ <position x="480.0" y="392.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<outputPort>
- <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
- <name>To enrichment</name>
- <position x="480.0" y="392.0" />
+ <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
+ <name>To data output</name>
+ <position x="-1120.0" y="592.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -4226,16 +4226,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
- <id>27d5761b-0172-1000-0000-000059275dad</id>
- <name>To enrichment</name>
- <position x="-312.0" y="328.0" />
+ <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
+ <name>To data output</name>
+ <position x="-632.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
<outputPort>
- <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
- <name>To data output</name>
- <position x="-632.0" y="328.0" />
+ <id>27d5761b-0172-1000-0000-000059275dad</id>
+ <name>To enrichment</name>
+ <position x="-312.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
@@ -4273,6 +4273,10 @@
<name>Routing Strategy</name>
<value>Route to Property name</value>
</property>
+ <property>
+ <name>keycloak</name>
+ <value>${log_type:equals("keycloak")}</value>
+ </property>
<property>
<name>kibana</name>
<value>${log_type:equals("kibana")}</value>
@@ -4295,16 +4299,16 @@
</property>
</processor>
<outputPort>
- <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
- <name>To enrichment</name>
- <position x="360.0" y="424.0" />
+ <id>bcb879d5-0175-1000-0000-000070879ad0</id>
+ <name>To data output</name>
+ <position x="-2480.0" y="336.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<outputPort>
- <id>bcb879d5-0175-1000-0000-000070879ad0</id>
- <name>To data output</name>
- <position x="-2480.0" y="336.0" />
+ <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
+ <name>To enrichment</name>
+ <position x="544.0" y="688.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -5104,14 +5108,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
- <position x="-1446.1517058240609" y="301.4492766741185" />
- </funnel>
<funnel>
<id>895faa7a-0175-1000-0000-000014ef9dd3</id>
<position x="278.84829417593915" y="332.4492766741185" />
</funnel>
+ <funnel>
+ <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
+ <position x="-1446.1517058240609" y="301.4492766741185" />
+ </funnel>
<connection>
<id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
<name />
@@ -5582,14 +5586,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
- <position x="283.72871497338747" y="273.4623850295515" />
- </funnel>
<funnel>
<id>d8f19295-5666-31a8-b701-52214c4db51d</id>
<position x="-1500.995244929405" y="257.20806784146276" />
</funnel>
+ <funnel>
+ <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
+ <position x="283.72871497338747" y="273.4623850295515" />
+ </funnel>
<processGroup>
<id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id>
<name>TLS events</name>
@@ -6102,14 +6106,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>8d399854-0175-1000-ffff-ffff8272837e</id>
- <position x="1736.0" y="528.0" />
- </funnel>
<funnel>
<id>8d3298f0-0175-1000-ffff-ffffc9f211a7</id>
<position x="56.0" y="280.0" />
</funnel>
+ <funnel>
+ <id>8d399854-0175-1000-ffff-ffff8272837e</id>
+ <position x="1736.0" y="528.0" />
+ </funnel>
<connection>
<id>8d3979b7-0175-1000-ffff-ffffe2efe898</id>
<name />
@@ -6924,6 +6928,190 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
+ <processGroup>
+ <id>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</id>
+ <name>Keycloak</name>
+ <position x="-440.0" y="1064.0" />
+ <comment />
+ <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
+ <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
+ <processor>
+ <id>8e17350e-583e-1130-8ec7-bd2dc5d4f361</id>
+ <name>UpdateAttribute</name>
+ <position x="344.0" y="736.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-keycloak</value>
+ </property>
+ </processor>
+ <processor>
+ <id>fbbe3f9c-5336-11c9-0000-00003ab5dde5</id>
+ <name>Fix timestamp</name>
+ <position x="352.0" y="480.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.jolt.record.JoltTransformRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-jolt-record-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>jolt-record-record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>jolt-record-record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ </property>
+ <property>
+ <name>jolt-record-transform</name>
+ <value>jolt-transform-chain</value>
+ </property>
+ <property>
+ <name>jolt-record-custom-class</name>
+ </property>
+ <property>
+ <name>jolt-record-custom-modules</name>
+ </property>
+ <property>
+ <name>jolt-record-spec</name>
+ <value>[
+ {
+ "operation": "shift",
+ "spec": {
+ "timestamp": {
+ "1": "timestamp"
+ },
+ "*": "&"
+ }
+ }
+]</value>
+ </property>
+ <property>
+ <name>jolt-record-transform-cache-size</name>
+ <value>1</value>
+ </property>
+ <autoTerminatedRelationship>failure</autoTerminatedRelationship>
+ <autoTerminatedRelationship>original</autoTerminatedRelationship>
+ </processor>
+ <inputPort>
+ <id>10cb3b64-e867-1d81-bd59-eb9cf6883f24</id>
+ <name>Input</name>
+ <position x="408.0" y="320.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </inputPort>
+ <outputPort>
+ <id>84dc3511-1322-175b-8083-9729037f8edb</id>
+ <name>Output</name>
+ <position x="392.0" y="984.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </outputPort>
+ <connection>
+ <id>fbbe3fbf-5336-11c9-ffff-ffffb7c3576e</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>fbbe3f9c-5336-11c9-0000-00003ab5dde5</sourceId>
+ <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8e17350e-583e-1130-8ec7-bd2dc5d4f361</destinationId>
+ <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>50c83129-28e1-1d45-bafe-912df3cdf284</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>10cb3b64-e867-1d81-bd59-eb9cf6883f24</sourceId>
+ <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>fbbe3f9c-5336-11c9-0000-00003ab5dde5</destinationId>
+ <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>fbbe3ede-5336-11c9-8870-deb7fffd14ae</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8e17350e-583e-1130-8ec7-bd2dc5d4f361</sourceId>
+ <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>84dc3511-1322-175b-8083-9729037f8edb</destinationId>
+ <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
<processGroup>
<id>83691174-683f-3c7c-8526-8fc00397aee1</id>
<name>Zeek</name>
@@ -7140,14 +7328,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>06521038-335b-3139-839d-ab43a013ce03</id>
- <position x="-1557.869726298236" y="758.8984861527665" />
- </funnel>
<funnel>
<id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
<position x="248.5321508445502" y="703.4412774751572" />
</funnel>
+ <funnel>
+ <id>06521038-335b-3139-839d-ab43a013ce03</id>
+ <position x="-1557.869726298236" y="758.8984861527665" />
+ </funnel>
<connection>
<id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
<name />
@@ -7372,6 +7560,26 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
+ <connection>
+ <id>fbbe3f1b-5336-11c9-ffff-ffffd29d2f5c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>84dc3511-1322-175b-8083-9729037f8edb</sourceId>
+ <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+ <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
<name />
@@ -7432,6 +7640,29 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>fbbe3ee1-5336-11c9-ffff-ffffa7c97811</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-720.0" y="1016.0" />
+ <bendPoint x="-584.0" y="1152.0" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+ <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>10cb3b64-e867-1d81-bd59-eb9cf6883f24</destinationId>
+ <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship>keycloak</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>8d1fe825-0175-1000-ffff-fffff0505cdc</id>
<name />
@@ -8149,7 +8380,7 @@
</property>
<property>
<name>Password</name>
- <value>enc{2d7036ed427615cc0da2c105923da69609e9a5b2cfdf3ae7356c2fb11de6538a5393d363e717b6316763851a10ca5679}</value>
+ <value>enc{7f706f76bc019ad8a3c7700ec14f6d65035b47cfa70fce4d0aade0809ded55af0afc391ccf1744443ffa082fc97204a6}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
@@ -11161,7 +11392,7 @@
</property>
<property>
<name>Truststore Password</name>
- <value>enc{f1a53d9f8ccdcff528b762ffc26710276eb38abb97f6abe2fd3fb2e8779ca390}</value>
+ <value>enc{fb94647e0f686a70205e73bdc73eb6a28bdbcd74c3e169b4cd116dd6a7cc28f6}</value>
</property>
<property>
<name>Truststore Type</name>