Skip to content
Snippets Groups Projects
Commit abb76d61 authored by Arne Øslebø's avatar Arne Øslebø
Browse files

parsing of zookeeper logs

parent 533acdc7
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
roles/nifi/templates/flow.xml.j2
+ 414
− 3
View file @ abb76d61
...@@ -4301,11 +4301,15 @@ ...@@ -4301,11 +4301,15 @@
<name>nifi</name> <name>nifi</name>
<value>${log_type:equals("nifi")}</value> <value>${log_type:equals("nifi")}</value>
</property> </property>
<property>
<name>zookeeper</name>
<value>${log_type:equals("zookeeper")}</value>
</property>
</processor> </processor>
<outputPort> <outputPort>
<id>349b32fe-a821-1197-0000-00003a0b6fe5</id> <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
<name>To enrichment</name> <name>To enrichment</name>
<position x="632.0" y="776.0" /> <position x="744.0" y="920.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
...@@ -8100,6 +8104,327 @@ ...@@ -8100,6 +8104,327 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
</processGroup> </processGroup>
<processGroup>
<id>31b13b40-8e26-1798-9777-2272881c6031</id>
<name>Zookeeper</name>
<position x="-440.0" y="1488.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>b09b367b-060f-1e74-9a96-ca5ba5f88858</id>
<name>UpdateRecord</name>
<position x="352.0" y="472.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
<name>replacement-value-strategy</name>
<value>literal-value</value>
</property>
<property>
<name>/labels/source_host</name>
<value>${source_host}</value>
</property>
<property>
<name>/timestamp</name>
<value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
</property>
</processor>
<processor>
<id>dd3b361c-4e9c-158c-ba31-61006a0b21b7</id>
<name>UpdateAttribute</name>
<position x="1072.0" y="472.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-update-attribute-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Delete Attributes Expression</name>
</property>
<property>
<name>Store State</name>
<value>Do not store state</value>
</property>
<property>
<name>Stateful Variables Initial Value</name>
</property>
<property>
<name>canonical-value-lookup-cache-size</name>
<value>100</value>
</property>
<property>
<name>data_index</name>
<value>logs-zookeeper</value>
</property>
</processor>
<processor>
<id>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</id>
<name>Extract message</name>
<position x="352.0" y="280.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
<value>bc8e5957-0175-1000-0000-00003346421d</value>
</property>
<property>
<name>include-zero-record-flowfiles</name>
<value>true</value>
</property>
</processor>
<processor>
<id>38f03e95-db26-1287-be1a-4218f647596a</id>
<name>Convert to json</name>
<position x="1064.0" y="272.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>7504a565-0176-1000-ffff-ffff9c0f0741</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
<name>include-zero-record-flowfiles</name>
<value>false</value>
</property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>a77d3c33-9575-1926-b230-9cf1fca55e7e</id>
<name>Input</name>
<position x="397.9999517774115" y="110.99999315685733" />
<comments />
<scheduledState>RUNNING</scheduledState>
</inputPort>
<outputPort>
<id>f1e73393-065a-1450-9ad0-fd7cdb57853f</id>
<name>Output</name>
<position x="1120.0" y="808.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<funnel>
<id>8f433b46-ddc7-163b-a48e-2f4436a68b1e</id>
<position x="8.0" y="424.0" />
</funnel>
<connection>
<id>e00437b9-7705-1d24-8df5-ab6739913b89</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>b09b367b-060f-1e74-9a96-ca5ba5f88858</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>8f433b46-ddc7-163b-a48e-2f4436a68b1e</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>FUNNEL</destinationType>
<relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>75043a2e-2420-1bbd-aa6a-49903351258d</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>8f433b46-ddc7-163b-a48e-2f4436a68b1e</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>FUNNEL</destinationType>
<relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>4b353d86-8a69-1ca1-bc1c-8db0049f5886</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>38f03e95-db26-1287-be1a-4218f647596a</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>b09b367b-060f-1e74-9a96-ca5ba5f88858</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>b05339e8-7bbc-1975-a9b2-4bc789dda2df</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>a77d3c33-9575-1926-b230-9cf1fca55e7e</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>5d6f3995-2075-11db-ba4b-1b76e1ed6473</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>b09b367b-060f-1e74-9a96-ca5ba5f88858</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>dd3b361c-4e9c-158c-ba31-61006a0b21b7</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>2a0131b7-9c0d-157d-a9ac-abd12398f2a8</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>38f03e95-db26-1287-be1a-4218f647596a</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>5a2e3db4-49d3-187d-9db9-3f0b48a2c6af</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>dd3b361c-4e9c-158c-ba31-61006a0b21b7</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>f1e73393-065a-1450-9ad0-fd7cdb57853f</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
<processGroup> <processGroup>
<id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id> <id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
<name>Kibana</name> <name>Kibana</name>
...@@ -8283,6 +8608,26 @@ ...@@ -8283,6 +8608,26 @@
<partitioningAttribute /> <partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection>
<id>75109cc1-0176-1000-ffff-ffff86db235d</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>f1e73393-065a-1450-9ad0-fd7cdb57853f</sourceId>
<sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
<destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection> <connection>
<id>8d0ea3d4-0175-1000-0000-0000471b8522</id> <id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
<name /> <name />
...@@ -8346,6 +8691,28 @@ ...@@ -8346,6 +8691,28 @@
<partitioningAttribute /> <partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection>
<id>74ff448b-0176-1000-0000-00002e302e83</id>
<name />
<bendPoints>
<bendPoint x="-688.0" y="1576.0" />
</bendPoints>
<labelIndex>0</labelIndex>
<zIndex>0</zIndex>
<sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
<sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>a77d3c33-9575-1926-b230-9cf1fca55e7e</destinationId>
<destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<relationship>zookeeper</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection> <connection>
<id>349b3398-a821-1197-ffff-ffffc5ae6471</id> <id>349b3398-a821-1197-ffff-ffffc5ae6471</id>
<name /> <name />
...@@ -9066,7 +9433,7 @@ ...@@ -9066,7 +9433,7 @@
</property> </property>
<property> <property>
<name>Password</name> <name>Password</name>
<value>enc{1c4c58f74f95b67649d56061742c742acc78f69ccc57c9e9ef50d02d079f56126e9174e3f3651d3f58f0fd58976866f6}</value> <value>enc{6a99599249debd0a65f95bddacbc6ca15fb19f869a4aabb9c432a18972b610121732f7f9643e9da95586b0a26f7268e0}</value>
</property> </property>
<property> <property>
<name>elasticsearch-http-connect-timeout</name> <name>elasticsearch-http-connect-timeout</name>
...@@ -11922,6 +12289,50 @@ ...@@ -11922,6 +12289,50 @@
<value>UTF-8</value> <value>UTF-8</value>
</property> </property>
</controllerService> </controllerService>
<controllerService>
<id>7504a565-0176-1000-ffff-ffff9c0f0741</id>
<name>Zookeeper logs</name>
<comment />
<class>org.apache.nifi.grok.GrokReader</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>schema-access-strategy</name>
<value>string-fields-from-grok-expression</value>
</property>
<property>
<name>schema-registry</name>
</property>
<property>
<name>schema-name</name>
<value>${schema.name}</value>
</property>
<property>
<name>schema-version</name>
</property>
<property>
<name>schema-branch</name>
</property>
<property>
<name>schema-text</name>
<value>${avro.schema}</value>
</property>
<property>
<name>Grok Pattern File</name>
</property>
<property>
<name>Grok Expression</name>
<value>%{GREEDYDATA:timestamp} \[%{DATA:id}\] - %{DATA:level} \[%{DATA:process}\] - %{GREEDYDATA:message}</value>
</property>
<property>
<name>no-match-behavior</name>
<value>append-to-previous-message</value>
</property>
</controllerService>
<controllerService> <controllerService>
<id>8b1dd8bb-0170-1000-0000-000007446e6a</id> <id>8b1dd8bb-0170-1000-0000-000007446e6a</id>
<name>Misp DistributedMapCacheServer</name> <name>Misp DistributedMapCacheServer</name>
...@@ -12138,7 +12549,7 @@ ...@@ -12138,7 +12549,7 @@
</property> </property>
<property> <property>
<name>Truststore Password</name> <name>Truststore Password</name>
<value>enc{9ca8dddd769be18fff9e22016c86d9f05a3a987f834c79a92609b7faa5e1e840}</value> <value>enc{d5f6f4e080eb19023e6034e50bb1beae8afe26c787f0dfd3e032857ec79dd535}</value>
</property> </property>
<property> <property>
<name>Truststore Type</name> <name>Truststore Type</name>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment