diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2
index bf07d53b3d82890ffd245ebfe1d6a0f4234b4315..05955862ecaf4490858a076be6d86ba3575832d7 100644
--- a/roles/nifi/templates/flow.xml.j2
+++ b/roles/nifi/templates/flow.xml.j2
@@ -4301,11 +4301,15 @@
<name>nifi</name>
<value>${log_type:equals("nifi")}</value>
</property>
+ <property>
+ <name>zookeeper</name>
+ <value>${log_type:equals("zookeeper")}</value>
+ </property>
</processor>
<outputPort>
<id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
<name>To enrichment</name>
- <position x="632.0" y="776.0" />
+ <position x="744.0" y="920.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -8100,6 +8104,327 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
+ <processGroup>
+ <id>31b13b40-8e26-1798-9777-2272881c6031</id>
+ <name>Zookeeper</name>
+ <position x="-440.0" y="1488.0" />
+ <comment />
+ <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
+ <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
+ <processor>
+ <id>b09b367b-060f-1e74-9a96-ca5ba5f88858</id>
+ <name>UpdateRecord</name>
+ <position x="352.0" y="472.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.UpdateRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ </property>
+ <property>
+ <name>replacement-value-strategy</name>
+ <value>literal-value</value>
+ </property>
+ <property>
+ <name>/labels/source_host</name>
+ <value>${source_host}</value>
+ </property>
+ <property>
+ <name>/timestamp</name>
+ <value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
+ </property>
+ </processor>
+ <processor>
+ <id>dd3b361c-4e9c-158c-ba31-61006a0b21b7</id>
+ <name>UpdateAttribute</name>
+ <position x="1072.0" y="472.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-zookeeper</value>
+ </property>
+ </processor>
+ <processor>
+ <id>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</id>
+ <name>Extract message</name>
+ <position x="352.0" y="280.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ConvertRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>bc8e5957-0175-1000-0000-00003346421d</value>
+ </property>
+ <property>
+ <name>include-zero-record-flowfiles</name>
+ <value>true</value>
+ </property>
+ </processor>
+ <processor>
+ <id>38f03e95-db26-1287-be1a-4218f647596a</id>
+ <name>Convert to json</name>
+ <position x="1064.0" y="272.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ConvertRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>7504a565-0176-1000-ffff-ffff9c0f0741</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ </property>
+ <property>
+ <name>include-zero-record-flowfiles</name>
+ <value>false</value>
+ </property>
+ <autoTerminatedRelationship>failure</autoTerminatedRelationship>
+ </processor>
+ <inputPort>
+ <id>a77d3c33-9575-1926-b230-9cf1fca55e7e</id>
+ <name>Input</name>
+ <position x="397.9999517774115" y="110.99999315685733" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </inputPort>
+ <outputPort>
+ <id>f1e73393-065a-1450-9ad0-fd7cdb57853f</id>
+ <name>Output</name>
+ <position x="1120.0" y="808.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </outputPort>
+ <funnel>
+ <id>8f433b46-ddc7-163b-a48e-2f4436a68b1e</id>
+ <position x="8.0" y="424.0" />
+ </funnel>
+ <connection>
+ <id>e00437b9-7705-1d24-8df5-ab6739913b89</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>b09b367b-060f-1e74-9a96-ca5ba5f88858</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8f433b46-ddc7-163b-a48e-2f4436a68b1e</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>60 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>75043a2e-2420-1bbd-aa6a-49903351258d</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8f433b46-ddc7-163b-a48e-2f4436a68b1e</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>60 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>4b353d86-8a69-1ca1-bc1c-8db0049f5886</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>38f03e95-db26-1287-be1a-4218f647596a</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>b09b367b-060f-1e74-9a96-ca5ba5f88858</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>b05339e8-7bbc-1975-a9b2-4bc789dda2df</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>a77d3c33-9575-1926-b230-9cf1fca55e7e</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5d6f3995-2075-11db-ba4b-1b76e1ed6473</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>b09b367b-060f-1e74-9a96-ca5ba5f88858</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>dd3b361c-4e9c-158c-ba31-61006a0b21b7</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>2a0131b7-9c0d-157d-a9ac-abd12398f2a8</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>38f03e95-db26-1287-be1a-4218f647596a</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5a2e3db4-49d3-187d-9db9-3f0b48a2c6af</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>dd3b361c-4e9c-158c-ba31-61006a0b21b7</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>f1e73393-065a-1450-9ad0-fd7cdb57853f</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
<processGroup>
<id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
<name>Kibana</name>
@@ -8283,6 +8608,26 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>75109cc1-0176-1000-ffff-ffff86db235d</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>f1e73393-065a-1450-9ad0-fd7cdb57853f</sourceId>
+ <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+ <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
<name />
@@ -8346,6 +8691,28 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>74ff448b-0176-1000-0000-00002e302e83</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-688.0" y="1576.0" />
+ </bendPoints>
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+ <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>a77d3c33-9575-1926-b230-9cf1fca55e7e</destinationId>
+ <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship>zookeeper</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>349b3398-a821-1197-ffff-ffffc5ae6471</id>
<name />
@@ -9066,7 +9433,7 @@
</property>
<property>
<name>Password</name>
- <value>enc{1c4c58f74f95b67649d56061742c742acc78f69ccc57c9e9ef50d02d079f56126e9174e3f3651d3f58f0fd58976866f6}</value>
+ <value>enc{6a99599249debd0a65f95bddacbc6ca15fb19f869a4aabb9c432a18972b610121732f7f9643e9da95586b0a26f7268e0}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
@@ -11922,6 +12289,50 @@
<value>UTF-8</value>
</property>
</controllerService>
+ <controllerService>
+ <id>7504a565-0176-1000-ffff-ffff9c0f0741</id>
+ <name>Zookeeper logs</name>
+ <comment />
+ <class>org.apache.nifi.grok.GrokReader</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-record-serialization-services-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <enabled>true</enabled>
+ <property>
+ <name>schema-access-strategy</name>
+ <value>string-fields-from-grok-expression</value>
+ </property>
+ <property>
+ <name>schema-registry</name>
+ </property>
+ <property>
+ <name>schema-name</name>
+ <value>${schema.name}</value>
+ </property>
+ <property>
+ <name>schema-version</name>
+ </property>
+ <property>
+ <name>schema-branch</name>
+ </property>
+ <property>
+ <name>schema-text</name>
+ <value>${avro.schema}</value>
+ </property>
+ <property>
+ <name>Grok Pattern File</name>
+ </property>
+ <property>
+ <name>Grok Expression</name>
+ <value>%{GREEDYDATA:timestamp} \[%{DATA:id}\] - %{DATA:level} \[%{DATA:process}\] - %{GREEDYDATA:message}</value>
+ </property>
+ <property>
+ <name>no-match-behavior</name>
+ <value>append-to-previous-message</value>
+ </property>
+ </controllerService>
<controllerService>
<id>8b1dd8bb-0170-1000-0000-000007446e6a</id>
<name>Misp DistributedMapCacheServer</name>
@@ -12138,7 +12549,7 @@
</property>
<property>
<name>Truststore Password</name>
- <value>enc{9ca8dddd769be18fff9e22016c86d9f05a3a987f834c79a92609b7faa5e1e840}</value>
+ <value>enc{d5f6f4e080eb19023e6034e50bb1beae8afe26c787f0dfd3e032857ec79dd535}</value>
</property>
<property>
<name>Truststore Type</name>