parsing of keycloak logs

inventories/filebeat

@@ -6,7 +6,7 @@ soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/l
 #soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="text"
 #soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text"
 soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
-soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="text"
+soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
 soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
 soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
 soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="zookeeper" FILEBEAT_LOG_FORMAT="text"

roles/keycloak/tasks/main.yml

@@ -36,6 +36,33 @@
   tags:
     - start
 
+- name: Configure logging format
+  remote_user: jboss
+  lineinfile: #TODO: Change to community.general.xml
+   path: /opt/jboss/keycloak/standalone/configuration/standalone.xml
+   regexp: '.*<formatter name="PATTERN">.*'
+   line: "<formatter name=\"JSON\"><json-formatter date-format=\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\" pretty-print=\"false\" print-details=\"true\" zone-id=\"UTC\"/></formatter><formatter name=\"PATTERN\">"
+  tags:
+   - start
+
+- name: Enable event logging
+  remote_user: jboss
+  lineinfile:
+   path: /opt/jboss/keycloak/standalone/configuration/standalone.xml
+   regexp: '.*<spi name="eventsStore">.*'
+   line: '<spi name="eventsListener"><provider name="jboss-logging" enabled="true"><properties><property name="success-level" value="INFO"/><property name="error-level" value="WARN"/></properties></provider></spi><spi name="eventsStore">'
+  tags:
+   - start
+
+- name: Specify logging format
+  remote_user: jboss
+  lineinfile:
+   path: /opt/jboss/keycloak/standalone/configuration/standalone.xml
+   regexp: ".*<named-formatter name=.*"
+   line: '<named-formatter name="JSON"/>'
+  tags:
+   - start
+
 - name: Configure Keycloak start script
   remote_user: jboss
   template:
@@ -48,7 +75,6 @@
   tags:
     - start
 
-
 - name: Start Keycloak IdP
   remote_user: root
   command: "supervisorctl start keycloak"

roles/nifi/templates/flow.xml.j2

@@ -4205,16 +4205,16 @@
         <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
         <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
         <outputPort>
-          <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
-          <name>To data output</name>
-          <position x="-1120.0" y="592.0" />
+          <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
+          <name>To enrichment</name>
+          <position x="480.0" y="392.0" />
           <comments />
           <scheduledState>RUNNING</scheduledState>
         </outputPort>
         <outputPort>
-          <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
-          <name>To enrichment</name>
-          <position x="480.0" y="392.0" />
+          <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
+          <name>To data output</name>
+          <position x="-1120.0" y="592.0" />
           <comments />
           <scheduledState>RUNNING</scheduledState>
         </outputPort>
@@ -4226,16 +4226,16 @@
           <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
           <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
           <outputPort>
-            <id>27d5761b-0172-1000-0000-000059275dad</id>
-            <name>To enrichment</name>
-            <position x="-312.0" y="328.0" />
+            <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
+            <name>To data output</name>
+            <position x="-632.0" y="328.0" />
             <comments />
             <scheduledState>STOPPED</scheduledState>
           </outputPort>
           <outputPort>
-            <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
-            <name>To data output</name>
-            <position x="-632.0" y="328.0" />
+            <id>27d5761b-0172-1000-0000-000059275dad</id>
+            <name>To enrichment</name>
+            <position x="-312.0" y="328.0" />
             <comments />
             <scheduledState>STOPPED</scheduledState>
           </outputPort>
@@ -4273,6 +4273,10 @@
               <name>Routing Strategy</name>
               <value>Route to Property name</value>
             </property>
+            <property>
+              <name>keycloak</name>
+              <value>${log_type:equals("keycloak")}</value>
+            </property>
             <property>
               <name>kibana</name>
               <value>${log_type:equals("kibana")}</value>
@@ -4295,16 +4299,16 @@
             </property>
           </processor>
           <outputPort>
-            <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
-            <name>To enrichment</name>
-            <position x="360.0" y="424.0" />
+            <id>bcb879d5-0175-1000-0000-000070879ad0</id>
+            <name>To data output</name>
+            <position x="-2480.0" y="336.0" />
             <comments />
             <scheduledState>RUNNING</scheduledState>
           </outputPort>
           <outputPort>
-            <id>bcb879d5-0175-1000-0000-000070879ad0</id>
-            <name>To data output</name>
-            <position x="-2480.0" y="336.0" />
+            <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
+            <name>To enrichment</name>
+            <position x="544.0" y="688.0" />
             <comments />
             <scheduledState>RUNNING</scheduledState>
           </outputPort>
@@ -5104,14 +5108,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
-              <position x="-1446.1517058240609" y="301.4492766741185" />
-            </funnel>
             <funnel>
               <id>895faa7a-0175-1000-0000-000014ef9dd3</id>
               <position x="278.84829417593915" y="332.4492766741185" />
             </funnel>
+            <funnel>
+              <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
+              <position x="-1446.1517058240609" y="301.4492766741185" />
+            </funnel>
             <connection>
               <id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
               <name />
@@ -5582,14 +5586,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
-              <position x="283.72871497338747" y="273.4623850295515" />
-            </funnel>
             <funnel>
               <id>d8f19295-5666-31a8-b701-52214c4db51d</id>
               <position x="-1500.995244929405" y="257.20806784146276" />
             </funnel>
+            <funnel>
+              <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
+              <position x="283.72871497338747" y="273.4623850295515" />
+            </funnel>
             <processGroup>
               <id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id>
               <name>TLS events</name>
@@ -6102,14 +6106,14 @@
                 <comments />
                 <scheduledState>RUNNING</scheduledState>
               </outputPort>
-              <funnel>
-                <id>8d399854-0175-1000-ffff-ffff8272837e</id>
-                <position x="1736.0" y="528.0" />
-              </funnel>
               <funnel>
                 <id>8d3298f0-0175-1000-ffff-ffffc9f211a7</id>
                 <position x="56.0" y="280.0" />
               </funnel>
+              <funnel>
+                <id>8d399854-0175-1000-ffff-ffff8272837e</id>
+                <position x="1736.0" y="528.0" />
+              </funnel>
               <connection>
                 <id>8d3979b7-0175-1000-ffff-ffffe2efe898</id>
                 <name />
@@ -6924,6 +6928,190 @@
               <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
             </connection>
           </processGroup>
+          <processGroup>
+            <id>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</id>
+            <name>Keycloak</name>
+            <position x="-440.0" y="1064.0" />
+            <comment />
+            <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
+            <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
+            <processor>
+              <id>8e17350e-583e-1130-8ec7-bd2dc5d4f361</id>
+              <name>UpdateAttribute</name>
+              <position x="344.0" y="736.0" />
+              <styles />
+              <comment />
+              <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+              <bundle>
+                <group>org.apache.nifi</group>
+                <artifact>nifi-update-attribute-nar</artifact>
+                <version>1.12.1</version>
+              </bundle>
+              <maxConcurrentTasks>1</maxConcurrentTasks>
+              <schedulingPeriod>0 sec</schedulingPeriod>
+              <penalizationPeriod>30 sec</penalizationPeriod>
+              <yieldPeriod>1 sec</yieldPeriod>
+              <bulletinLevel>WARN</bulletinLevel>
+              <lossTolerant>false</lossTolerant>
+              <scheduledState>RUNNING</scheduledState>
+              <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+              <executionNode>ALL</executionNode>
+              <runDurationNanos>0</runDurationNanos>
+              <property>
+                <name>Delete Attributes Expression</name>
+              </property>
+              <property>
+                <name>Store State</name>
+                <value>Do not store state</value>
+              </property>
+              <property>
+                <name>Stateful Variables Initial Value</name>
+              </property>
+              <property>
+                <name>canonical-value-lookup-cache-size</name>
+                <value>100</value>
+              </property>
+              <property>
+                <name>data_index</name>
+                <value>logs-keycloak</value>
+              </property>
+            </processor>
+            <processor>
+              <id>fbbe3f9c-5336-11c9-0000-00003ab5dde5</id>
+              <name>Fix timestamp</name>
+              <position x="352.0" y="480.0" />
+              <styles />
+              <comment />
+              <class>org.apache.nifi.processors.jolt.record.JoltTransformRecord</class>
+              <bundle>
+                <group>org.apache.nifi</group>
+                <artifact>nifi-jolt-record-nar</artifact>
+                <version>1.12.1</version>
+              </bundle>
+              <maxConcurrentTasks>1</maxConcurrentTasks>
+              <schedulingPeriod>0 sec</schedulingPeriod>
+              <penalizationPeriod>30 sec</penalizationPeriod>
+              <yieldPeriod>1 sec</yieldPeriod>
+              <bulletinLevel>WARN</bulletinLevel>
+              <lossTolerant>false</lossTolerant>
+              <scheduledState>RUNNING</scheduledState>
+              <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+              <executionNode>ALL</executionNode>
+              <runDurationNanos>0</runDurationNanos>
+              <property>
+                <name>jolt-record-record-reader</name>
+                <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+              </property>
+              <property>
+                <name>jolt-record-record-writer</name>
+                <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+              </property>
+              <property>
+                <name>jolt-record-transform</name>
+                <value>jolt-transform-chain</value>
+              </property>
+              <property>
+                <name>jolt-record-custom-class</name>
+              </property>
+              <property>
+                <name>jolt-record-custom-modules</name>
+              </property>
+              <property>
+                <name>jolt-record-spec</name>
+                <value>[
+  {
+    "operation": "shift",
+    "spec": {
+      "timestamp": {
+        "1": "timestamp"
+      },
+      "*": "&amp;"
+    }
+  }
+]</value>
+              </property>
+              <property>
+                <name>jolt-record-transform-cache-size</name>
+                <value>1</value>
+              </property>
+              <autoTerminatedRelationship>failure</autoTerminatedRelationship>
+              <autoTerminatedRelationship>original</autoTerminatedRelationship>
+            </processor>
+            <inputPort>
+              <id>10cb3b64-e867-1d81-bd59-eb9cf6883f24</id>
+              <name>Input</name>
+              <position x="408.0" y="320.0" />
+              <comments />
+              <scheduledState>RUNNING</scheduledState>
+            </inputPort>
+            <outputPort>
+              <id>84dc3511-1322-175b-8083-9729037f8edb</id>
+              <name>Output</name>
+              <position x="392.0" y="984.0" />
+              <comments />
+              <scheduledState>RUNNING</scheduledState>
+            </outputPort>
+            <connection>
+              <id>fbbe3fbf-5336-11c9-ffff-ffffb7c3576e</id>
+              <name />
+              <bendPoints />
+              <labelIndex>1</labelIndex>
+              <zIndex>0</zIndex>
+              <sourceId>fbbe3f9c-5336-11c9-0000-00003ab5dde5</sourceId>
+              <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+              <sourceType>PROCESSOR</sourceType>
+              <destinationId>8e17350e-583e-1130-8ec7-bd2dc5d4f361</destinationId>
+              <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+              <destinationType>PROCESSOR</destinationType>
+              <relationship>success</relationship>
+              <maxWorkQueueSize>10000</maxWorkQueueSize>
+              <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+              <flowFileExpiration>0 sec</flowFileExpiration>
+              <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+              <partitioningAttribute />
+              <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+            </connection>
+            <connection>
+              <id>50c83129-28e1-1d45-bafe-912df3cdf284</id>
+              <name />
+              <bendPoints />
+              <labelIndex>1</labelIndex>
+              <zIndex>0</zIndex>
+              <sourceId>10cb3b64-e867-1d81-bd59-eb9cf6883f24</sourceId>
+              <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+              <sourceType>INPUT_PORT</sourceType>
+              <destinationId>fbbe3f9c-5336-11c9-0000-00003ab5dde5</destinationId>
+              <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+              <destinationType>PROCESSOR</destinationType>
+              <relationship />
+              <maxWorkQueueSize>10000</maxWorkQueueSize>
+              <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+              <flowFileExpiration>0 sec</flowFileExpiration>
+              <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+              <partitioningAttribute />
+              <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+            </connection>
+            <connection>
+              <id>fbbe3ede-5336-11c9-8870-deb7fffd14ae</id>
+              <name />
+              <bendPoints />
+              <labelIndex>1</labelIndex>
+              <zIndex>0</zIndex>
+              <sourceId>8e17350e-583e-1130-8ec7-bd2dc5d4f361</sourceId>
+              <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+              <sourceType>PROCESSOR</sourceType>
+              <destinationId>84dc3511-1322-175b-8083-9729037f8edb</destinationId>
+              <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+              <destinationType>OUTPUT_PORT</destinationType>
+              <relationship>success</relationship>
+              <maxWorkQueueSize>10000</maxWorkQueueSize>
+              <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+              <flowFileExpiration>0 sec</flowFileExpiration>
+              <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+              <partitioningAttribute />
+              <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+            </connection>
+          </processGroup>
           <processGroup>
             <id>83691174-683f-3c7c-8526-8fc00397aee1</id>
             <name>Zeek</name>
@@ -7140,14 +7328,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>06521038-335b-3139-839d-ab43a013ce03</id>
-              <position x="-1557.869726298236" y="758.8984861527665" />
-            </funnel>
             <funnel>
               <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
               <position x="248.5321508445502" y="703.4412774751572" />
             </funnel>
+            <funnel>
+              <id>06521038-335b-3139-839d-ab43a013ce03</id>
+              <position x="-1557.869726298236" y="758.8984861527665" />
+            </funnel>
             <connection>
               <id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
               <name />
@@ -7372,6 +7560,26 @@
               <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
             </connection>
           </processGroup>
+          <connection>
+            <id>fbbe3f1b-5336-11c9-ffff-ffffd29d2f5c</id>
+            <name />
+            <bendPoints />
+            <labelIndex>1</labelIndex>
+            <zIndex>0</zIndex>
+            <sourceId>84dc3511-1322-175b-8083-9729037f8edb</sourceId>
+            <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
+            <sourceType>OUTPUT_PORT</sourceType>
+            <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+            <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+            <destinationType>OUTPUT_PORT</destinationType>
+            <relationship />
+            <maxWorkQueueSize>10000</maxWorkQueueSize>
+            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+            <flowFileExpiration>0 sec</flowFileExpiration>
+            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+            <partitioningAttribute />
+            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+          </connection>
           <connection>
             <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
             <name />
@@ -7432,6 +7640,29 @@
             <partitioningAttribute />
             <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
           </connection>
+          <connection>
+            <id>fbbe3ee1-5336-11c9-ffff-ffffa7c97811</id>
+            <name />
+            <bendPoints>
+              <bendPoint x="-720.0" y="1016.0" />
+              <bendPoint x="-584.0" y="1152.0" />
+            </bendPoints>
+            <labelIndex>1</labelIndex>
+            <zIndex>0</zIndex>
+            <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+            <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+            <sourceType>PROCESSOR</sourceType>
+            <destinationId>10cb3b64-e867-1d81-bd59-eb9cf6883f24</destinationId>
+            <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
+            <destinationType>INPUT_PORT</destinationType>
+            <relationship>keycloak</relationship>
+            <maxWorkQueueSize>10000</maxWorkQueueSize>
+            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+            <flowFileExpiration>0 sec</flowFileExpiration>
+            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+            <partitioningAttribute />
+            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+          </connection>
           <connection>
             <id>8d1fe825-0175-1000-ffff-fffff0505cdc</id>
             <name />
@@ -8149,7 +8380,7 @@
             </property>
             <property>
               <name>Password</name>
-              <value>enc{2d7036ed427615cc0da2c105923da69609e9a5b2cfdf3ae7356c2fb11de6538a5393d363e717b6316763851a10ca5679}</value>
+              <value>enc{7f706f76bc019ad8a3c7700ec14f6d65035b47cfa70fce4d0aade0809ded55af0afc391ccf1744443ffa082fc97204a6}</value>
             </property>
             <property>
               <name>elasticsearch-http-connect-timeout</name>
@@ -11161,7 +11392,7 @@
       </property>
       <property>
         <name>Truststore Password</name>
-        <value>enc{f1a53d9f8ccdcff528b762ffc26710276eb38abb97f6abe2fd3fb2e8779ca390}</value>
+        <value>enc{fb94647e0f686a70205e73bdc73eb6a28bdbcd74c3e169b4cd116dd6a7cc28f6}</value>
       </property>
       <property>
         <name>Truststore Type</name>