added parsing of kibana logs

inventories/filebeat

@@ -5,7 +5,7 @@ soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
 soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
 #soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe1" FILEBEAT_LOG_FORMAT="text"
 #soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text"
-soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="text"
+soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
 soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="text"
 soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
 soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"

roles/nifi/templates/flow.xml.j2

@@ -4226,16 +4226,16 @@
           <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
           <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
           <outputPort>
-            <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
-            <name>To data output</name>
-            <position x="-632.0" y="328.0" />
+            <id>27d5761b-0172-1000-0000-000059275dad</id>
+            <name>To enrichment</name>
+            <position x="-312.0" y="328.0" />
             <comments />
             <scheduledState>STOPPED</scheduledState>
           </outputPort>
           <outputPort>
-            <id>27d5761b-0172-1000-0000-000059275dad</id>
-            <name>To enrichment</name>
-            <position x="-312.0" y="328.0" />
+            <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
+            <name>To data output</name>
+            <position x="-632.0" y="328.0" />
             <comments />
             <scheduledState>STOPPED</scheduledState>
           </outputPort>
@@ -4273,6 +4273,10 @@
               <name>Routing Strategy</name>
               <value>Route to Property name</value>
             </property>
+            <property>
+              <name>kibana</name>
+              <value>${log_type:equals("kibana")}</value>
+            </property>
             <property>
               <name>suricata</name>
               <value>${log_type:equals("suricata")}</value>
@@ -5100,14 +5104,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>895faa7a-0175-1000-0000-000014ef9dd3</id>
-              <position x="278.84829417593915" y="332.4492766741185" />
-            </funnel>
             <funnel>
               <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
               <position x="-1446.1517058240609" y="301.4492766741185" />
             </funnel>
+            <funnel>
+              <id>895faa7a-0175-1000-0000-000014ef9dd3</id>
+              <position x="278.84829417593915" y="332.4492766741185" />
+            </funnel>
             <connection>
               <id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
               <name />
@@ -6732,6 +6736,10 @@
                 <name>data_index</name>
                 <value>logs-haproxy</value>
               </property>
+              <property>
+                <name>enrich_ip1</name>
+                <value>/client.ip</value>
+              </property>
             </processor>
             <inputPort>
               <id>65a33e05-e157-1bfc-8741-adf11b3df720</id>
@@ -6747,14 +6755,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id>
-              <position x="1882.9999517774115" y="327.9999931568573" />
-            </funnel>
             <funnel>
               <id>312d3490-461e-13ac-a3a2-603704c456e2</id>
               <position x="8.0" y="424.0" />
             </funnel>
+            <funnel>
+              <id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id>
+              <position x="1882.9999517774115" y="327.9999931568573" />
+            </funnel>
             <connection>
               <id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id>
               <name />
@@ -7132,14 +7140,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
-              <position x="248.5321508445502" y="703.4412774751572" />
-            </funnel>
             <funnel>
               <id>06521038-335b-3139-839d-ab43a013ce03</id>
               <position x="-1557.869726298236" y="758.8984861527665" />
             </funnel>
+            <funnel>
+              <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
+              <position x="248.5321508445502" y="703.4412774751572" />
+            </funnel>
             <connection>
               <id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
               <name />
@@ -7261,6 +7269,169 @@
               <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
             </connection>
           </processGroup>
+          <processGroup>
+            <id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
+            <name>Kibana</name>
+            <position x="-440.0" y="864.0" />
+            <comment />
+            <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
+            <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
+            <processor>
+              <id>992c3710-1c87-169c-ab17-d2597387a25e</id>
+              <name>UpdateAttribute</name>
+              <position x="360.0" y="512.0" />
+              <styles />
+              <comment />
+              <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+              <bundle>
+                <group>org.apache.nifi</group>
+                <artifact>nifi-update-attribute-nar</artifact>
+                <version>1.12.1</version>
+              </bundle>
+              <maxConcurrentTasks>1</maxConcurrentTasks>
+              <schedulingPeriod>0 sec</schedulingPeriod>
+              <penalizationPeriod>30 sec</penalizationPeriod>
+              <yieldPeriod>1 sec</yieldPeriod>
+              <bulletinLevel>WARN</bulletinLevel>
+              <lossTolerant>false</lossTolerant>
+              <scheduledState>RUNNING</scheduledState>
+              <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+              <executionNode>ALL</executionNode>
+              <runDurationNanos>0</runDurationNanos>
+              <property>
+                <name>Delete Attributes Expression</name>
+              </property>
+              <property>
+                <name>Store State</name>
+                <value>Do not store state</value>
+              </property>
+              <property>
+                <name>Stateful Variables Initial Value</name>
+              </property>
+              <property>
+                <name>canonical-value-lookup-cache-size</name>
+                <value>100</value>
+              </property>
+              <property>
+                <name>data_index</name>
+                <value>logs-kibana</value>
+              </property>
+            </processor>
+            <inputPort>
+              <id>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</id>
+              <name>Input</name>
+              <position x="408.0" y="320.0" />
+              <comments />
+              <scheduledState>RUNNING</scheduledState>
+            </inputPort>
+            <outputPort>
+              <id>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</id>
+              <name>Output</name>
+              <position x="408.0" y="760.0" />
+              <comments />
+              <scheduledState>RUNNING</scheduledState>
+            </outputPort>
+            <connection>
+              <id>cc403fb4-8d68-1c68-82c3-b9af4affddaa</id>
+              <name />
+              <bendPoints />
+              <labelIndex>1</labelIndex>
+              <zIndex>0</zIndex>
+              <sourceId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</sourceId>
+              <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
+              <sourceType>INPUT_PORT</sourceType>
+              <destinationId>992c3710-1c87-169c-ab17-d2597387a25e</destinationId>
+              <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
+              <destinationType>PROCESSOR</destinationType>
+              <relationship />
+              <maxWorkQueueSize>10000</maxWorkQueueSize>
+              <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+              <flowFileExpiration>0 sec</flowFileExpiration>
+              <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+              <partitioningAttribute />
+              <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+            </connection>
+            <connection>
+              <id>b9e33c29-910f-134a-8390-2970800d7fcf</id>
+              <name />
+              <bendPoints />
+              <labelIndex>1</labelIndex>
+              <zIndex>0</zIndex>
+              <sourceId>992c3710-1c87-169c-ab17-d2597387a25e</sourceId>
+              <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
+              <sourceType>PROCESSOR</sourceType>
+              <destinationId>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</destinationId>
+              <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
+              <destinationType>OUTPUT_PORT</destinationType>
+              <relationship>success</relationship>
+              <maxWorkQueueSize>10000</maxWorkQueueSize>
+              <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+              <flowFileExpiration>0 sec</flowFileExpiration>
+              <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+              <partitioningAttribute />
+              <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+            </connection>
+          </processGroup>
+          <connection>
+            <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
+            <name />
+            <bendPoints />
+            <labelIndex>1</labelIndex>
+            <zIndex>0</zIndex>
+            <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId>
+            <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+            <sourceType>OUTPUT_PORT</sourceType>
+            <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+            <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+            <destinationType>OUTPUT_PORT</destinationType>
+            <relationship />
+            <maxWorkQueueSize>10000</maxWorkQueueSize>
+            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+            <flowFileExpiration>0 sec</flowFileExpiration>
+            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+            <partitioningAttribute />
+            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+          </connection>
+          <connection>
+            <id>349b339b-a821-1197-0000-00002e648df6</id>
+            <name />
+            <bendPoints />
+            <labelIndex>1</labelIndex>
+            <zIndex>0</zIndex>
+            <sourceId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</sourceId>
+            <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+            <sourceType>OUTPUT_PORT</sourceType>
+            <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+            <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+            <destinationType>OUTPUT_PORT</destinationType>
+            <relationship />
+            <maxWorkQueueSize>10000</maxWorkQueueSize>
+            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+            <flowFileExpiration>0 sec</flowFileExpiration>
+            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+            <partitioningAttribute />
+            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+          </connection>
+          <connection>
+            <id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
+            <name />
+            <bendPoints />
+            <labelIndex>1</labelIndex>
+            <zIndex>0</zIndex>
+            <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+            <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+            <sourceType>PROCESSOR</sourceType>
+            <destinationId>89639d3d-0175-1000-ffff-ffffb446c257</destinationId>
+            <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
+            <destinationType>INPUT_PORT</destinationType>
+            <relationship>unmatched</relationship>
+            <maxWorkQueueSize>10000</maxWorkQueueSize>
+            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+            <flowFileExpiration>0 sec</flowFileExpiration>
+            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+            <partitioningAttribute />
+            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+          </connection>
           <connection>
             <id>8d1fe825-0175-1000-ffff-fffff0505cdc</id>
             <name />
@@ -7301,6 +7472,26 @@
             <partitioningAttribute />
             <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
           </connection>
+          <connection>
+            <id>61c51cd8-0176-1000-ffff-ffff9247ba7c</id>
+            <name />
+            <bendPoints />
+            <labelIndex>1</labelIndex>
+            <zIndex>0</zIndex>
+            <sourceId>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</sourceId>
+            <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
+            <sourceType>OUTPUT_PORT</sourceType>
+            <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+            <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+            <destinationType>OUTPUT_PORT</destinationType>
+            <relationship />
+            <maxWorkQueueSize>10000</maxWorkQueueSize>
+            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+            <flowFileExpiration>0 sec</flowFileExpiration>
+            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+            <partitioningAttribute />
+            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+          </connection>
           <connection>
             <id>bc6e50cc-0175-1000-ffff-ffffbd982e0c</id>
             <name />
@@ -7344,18 +7535,20 @@
             <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
           </connection>
           <connection>
-            <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
+            <id>6196cd03-0176-1000-ffff-ffffd39b8c82</id>
             <name />
-            <bendPoints />
-            <labelIndex>1</labelIndex>
+            <bendPoints>
+              <bendPoint x="-576.0" y="896.0" />
+            </bendPoints>
+            <labelIndex>0</labelIndex>
             <zIndex>0</zIndex>
-            <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId>
-            <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
-            <sourceType>OUTPUT_PORT</sourceType>
-            <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
-            <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
-            <destinationType>OUTPUT_PORT</destinationType>
-            <relationship />
+            <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+            <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+            <sourceType>PROCESSOR</sourceType>
+            <destinationId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</destinationId>
+            <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
+            <destinationType>INPUT_PORT</destinationType>
+            <relationship>kibana</relationship>
             <maxWorkQueueSize>10000</maxWorkQueueSize>
             <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
             <flowFileExpiration>0 sec</flowFileExpiration>
@@ -7384,13 +7577,13 @@
             <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
           </connection>
           <connection>
-            <id>349b339b-a821-1197-0000-00002e648df6</id>
+            <id>349b3303-a821-1197-ffff-ffffa12b866d</id>
             <name />
             <bendPoints />
             <labelIndex>1</labelIndex>
             <zIndex>0</zIndex>
-            <sourceId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</sourceId>
-            <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+            <sourceId>8963b202-0175-1000-0000-000022d64ba2</sourceId>
+            <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
             <sourceType>OUTPUT_PORT</sourceType>
             <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
             <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
@@ -7423,46 +7616,6 @@
             <partitioningAttribute />
             <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
           </connection>
-          <connection>
-            <id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
-            <name />
-            <bendPoints />
-            <labelIndex>1</labelIndex>
-            <zIndex>0</zIndex>
-            <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
-            <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
-            <sourceType>PROCESSOR</sourceType>
-            <destinationId>89639d3d-0175-1000-ffff-ffffb446c257</destinationId>
-            <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
-            <destinationType>INPUT_PORT</destinationType>
-            <relationship>unmatched</relationship>
-            <maxWorkQueueSize>10000</maxWorkQueueSize>
-            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
-            <flowFileExpiration>0 sec</flowFileExpiration>
-            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
-            <partitioningAttribute />
-            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
-          </connection>
-          <connection>
-            <id>349b3303-a821-1197-ffff-ffffa12b866d</id>
-            <name />
-            <bendPoints />
-            <labelIndex>1</labelIndex>
-            <zIndex>0</zIndex>
-            <sourceId>8963b202-0175-1000-0000-000022d64ba2</sourceId>
-            <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
-            <sourceType>OUTPUT_PORT</sourceType>
-            <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
-            <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
-            <destinationType>OUTPUT_PORT</destinationType>
-            <relationship />
-            <maxWorkQueueSize>10000</maxWorkQueueSize>
-            <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
-            <flowFileExpiration>0 sec</flowFileExpiration>
-            <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
-            <partitioningAttribute />
-            <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
-          </connection>
           <connection>
             <id>349b3301-a821-1197-0000-0000070259c4</id>
             <name />
@@ -7996,7 +8149,7 @@
             </property>
             <property>
               <name>Password</name>
-              <value>enc{1c9a67efa861b9a5f0ced47e1bb930650b19b788b8576e55d87fa2a3a4760d790d7425f299ed70ea1859a64a26753959}</value>
+              <value>enc{2d7036ed427615cc0da2c105923da69609e9a5b2cfdf3ae7356c2fb11de6538a5393d363e717b6316763851a10ca5679}</value>
             </property>
             <property>
               <name>elasticsearch-http-connect-timeout</name>
@@ -11008,7 +11161,7 @@
       </property>
       <property>
         <name>Truststore Password</name>
-        <value>enc{d064a1e3a5a974d37b0202bbb9551137b9543af176d965ad630f0fc2bdafa690}</value>
+        <value>enc{f1a53d9f8ccdcff528b762ffc26710276eb38abb97f6abe2fd3fb2e8779ca390}</value>
       </property>
       <property>
         <name>Truststore Type</name>