Skip to content
Snippets Groups Projects
Commit 533acdc7 authored by Arne Øslebø's avatar Arne Øslebø
Browse files

turned on audit logging in mysql and parsing of logs in nifi

parent 0fd03849
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
inventories/filebeat
+ 1
− 1
View file @ 533acdc7
...@@ -7,7 +7,7 @@ soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/l ...@@ -7,7 +7,7 @@ soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/l
#soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text" #soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="odfe2" FILEBEAT_LOG_FORMAT="text"
soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json" soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json" soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text" soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log","/var/opt/rh/rh-mariadb103/lib/mysql/server_audit.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text" soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="zookeeper" FILEBEAT_LOG_FORMAT="text" soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="zookeeper" FILEBEAT_LOG_FORMAT="text"
soctools-cortex ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cortex" FILEBEAT_LOG_FORMAT="text" soctools-cortex ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cortex" FILEBEAT_LOG_FORMAT="text"
... ...
......
roles/mysql/templates/mysql_secure.sql.j2
+ 5
− 0
View file @ 533acdc7
...@@ -10,5 +10,10 @@ GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDE ...@@ -10,5 +10,10 @@ GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDE
GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}'; GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}';
{% endfor %} {% endfor %}
INSTALL SONAME 'server_audit';
SET GLOBAL server_audit_logging=ON;
SET GLOBAL server_audit_file_rotate_now = ON;
SET GLOBAl server_audit_file_rotations = 2;
SET GLOABL audit_log_format = JSON;
FLUSH PRIVILEGES; FLUSH PRIVILEGES;
roles/nifi/templates/flow.xml.j2
+ 1060
− 242
View file @ 533acdc7
...@@ -4205,16 +4205,16 @@ ...@@ -4205,16 +4205,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency> <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort> <outputPort>
<id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id> <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
<name>To enrichment</name> <name>To data output</name>
<position x="480.0" y="392.0" /> <position x="-1120.0" y="592.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
<outputPort> <outputPort>
<id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id> <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
<name>To data output</name> <name>To enrichment</name>
<position x="-1120.0" y="592.0" /> <position x="480.0" y="392.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
...@@ -4226,16 +4226,16 @@ ...@@ -4226,16 +4226,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency> <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort> <outputPort>
<id>27d5dab2-0172-1000-ffff-ffffab5c50be</id> <id>27d5761b-0172-1000-0000-000059275dad</id>
<name>To data output</name> <name>To enrichment</name>
<position x="-632.0" y="328.0" /> <position x="-312.0" y="328.0" />
<comments /> <comments />
<scheduledState>STOPPED</scheduledState> <scheduledState>STOPPED</scheduledState>
</outputPort> </outputPort>
<outputPort> <outputPort>
<id>27d5761b-0172-1000-0000-000059275dad</id> <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
<name>To enrichment</name> <name>To data output</name>
<position x="-312.0" y="328.0" /> <position x="-632.0" y="328.0" />
<comments /> <comments />
<scheduledState>STOPPED</scheduledState> <scheduledState>STOPPED</scheduledState>
</outputPort> </outputPort>
...@@ -4289,6 +4289,10 @@ ...@@ -4289,6 +4289,10 @@
<name>haproxy</name> <name>haproxy</name>
<value>${log_type:equals("haproxy")}</value> <value>${log_type:equals("haproxy")}</value>
</property> </property>
<property>
<name>mysql</name>
<value>${log_type:equals("mysql")}</value>
</property>
<property> <property>
<name>zeek</name> <name>zeek</name>
<value>${log_type:equals("zeek")}</value> <value>${log_type:equals("zeek")}</value>
...@@ -4299,16 +4303,16 @@ ...@@ -4299,16 +4303,16 @@
</property> </property>
</processor> </processor>
<outputPort> <outputPort>
<id>bcb879d5-0175-1000-0000-000070879ad0</id> <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
<name>To data output</name> <name>To enrichment</name>
<position x="-2480.0" y="336.0" /> <position x="632.0" y="776.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
<outputPort> <outputPort>
<id>349b32fe-a821-1197-0000-00003a0b6fe5</id> <id>bcb879d5-0175-1000-0000-000070879ad0</id>
<name>To enrichment</name> <name>To data output</name>
<position x="544.0" y="688.0" /> <position x="-2480.0" y="336.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
...@@ -4594,14 +4598,14 @@ ...@@ -4594,14 +4598,14 @@
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
<funnel>
<id>bc90d189-0175-1000-0000-0000037bc986</id>
<position x="8.0" y="424.0" />
</funnel>
<funnel> <funnel>
<id>bc925474-0175-1000-0000-00004e78071f</id> <id>bc925474-0175-1000-0000-00004e78071f</id>
<position x="1882.9999517774115" y="327.9999931568573" /> <position x="1882.9999517774115" y="327.9999931568573" />
</funnel> </funnel>
<funnel>
<id>bc90d189-0175-1000-0000-0000037bc986</id>
<position x="8.0" y="424.0" />
</funnel>
<connection> <connection>
<id>bc90c7ac-0175-1000-ffff-fffffa80b534</id> <id>bc90c7ac-0175-1000-ffff-fffffa80b534</id>
<name /> <name />
...@@ -5108,14 +5112,14 @@ ...@@ -5108,14 +5112,14 @@
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
<funnel>
<id>895faa7a-0175-1000-0000-000014ef9dd3</id>
<position x="278.84829417593915" y="332.4492766741185" />
</funnel>
<funnel> <funnel>
<id>895f7db3-0175-1000-ffff-ffff8229d688</id> <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
<position x="-1446.1517058240609" y="301.4492766741185" /> <position x="-1446.1517058240609" y="301.4492766741185" />
</funnel> </funnel>
<funnel>
<id>895faa7a-0175-1000-0000-000014ef9dd3</id>
<position x="278.84829417593915" y="332.4492766741185" />
</funnel>
<connection> <connection>
<id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id> <id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
<name /> <name />
...@@ -6106,14 +6110,14 @@ ...@@ -6106,14 +6110,14 @@
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
<funnel>
<id>8d3298f0-0175-1000-ffff-ffffc9f211a7</id>
<position x="56.0" y="280.0" />
</funnel>
<funnel> <funnel>
<id>8d399854-0175-1000-ffff-ffff8272837e</id> <id>8d399854-0175-1000-ffff-ffff8272837e</id>
<position x="1736.0" y="528.0" /> <position x="1736.0" y="528.0" />
</funnel> </funnel>
<funnel>
<id>8d3298f0-0175-1000-ffff-ffffc9f211a7</id>
<position x="56.0" y="280.0" />
</funnel>
<connection> <connection>
<id>8d3979b7-0175-1000-ffff-ffffe2efe898</id> <id>8d3979b7-0175-1000-ffff-ffffe2efe898</id>
<name /> <name />
...@@ -6581,14 +6585,114 @@ ...@@ -6581,14 +6585,114 @@
</connection> </connection>
</processGroup> </processGroup>
<processGroup> <processGroup>
<id>5d04357e-423c-1ab5-a7a4-44565abfed7f</id> <id>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</id>
<name>Haproxy</name> <name>Mysql</name>
<position x="-448.0" y="664.0" /> <position x="-440.0" y="1272.0" />
<comment /> <comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency> <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor> <processor>
<id>c9763c4c-7186-1460-871a-b5fd00ca3241</id> <id>14453e90-7646-1485-ffff-ffff81f3c683</id>
<name>Add header</name>
<position x="344.0" y="-8.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ReplaceText</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Regular Expression</name>
<value>(?s)(^.*$)</value>
</property>
<property>
<name>Replacement Value</name>
<value>timestamp,serverhost,username,host,connectionid,queryid,operation,database,object,retcode
</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>Maximum Buffer Size</name>
<value>1 MB</value>
</property>
<property>
<name>Replacement Strategy</name>
<value>Prepend</value>
</property>
<property>
<name>Evaluation Mode</name>
<value>Entire text</value>
</property>
<property>
<name>Line-by-Line Evaluation Mode</name>
<value>All</value>
</property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>e0bd3907-2d13-1407-b2dd-48591e65e59d</id>
<name>UpdateRecord</name>
<position x="-336.0" y="416.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
<name>replacement-value-strategy</name>
<value>literal-value</value>
</property>
<property>
<name>/event_type</name>
<value>log</value>
</property>
<property>
<name>/labels/source_host</name>
<value>${source_host}</value>
</property>
<property>
<name>/timestamp</name>
<value>${field.value:toDate('yyMMdd HH:mm:ss'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
</property>
</processor>
<processor>
<id>50813f6b-a5f6-1a98-8ae4-115134714332</id>
<name>UpdateRecord</name> <name>UpdateRecord</name>
<position x="352.0" y="472.0" /> <position x="352.0" y="472.0" />
<styles /> <styles />
...@@ -6621,19 +6725,54 @@ ...@@ -6621,19 +6725,54 @@
<name>replacement-value-strategy</name> <name>replacement-value-strategy</name>
<value>literal-value</value> <value>literal-value</value>
</property> </property>
<property>
<name>/event_type</name>
<value>audit</value>
</property>
<property> <property>
<name>/labels/source_host</name> <name>/labels/source_host</name>
<value>${source_host}</value> <value>${source_host}</value>
</property> </property>
<property> <property>
<name>/timestamp</name> <name>/timestamp</name>
<value>${field.value:toDate('dd/MMM/yyyy:HH:mm:ss.SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value> <value>${field.value:toDate('yyyyMMdd HH:mm:ss'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
</property> </property>
</processor> </processor>
<processor> <processor>
<id>e4c8356d-54ad-15b5-94fe-799d9465aa51</id> <id>e4353681-23e9-15af-0000-000032ea35e3</id>
<name>RouteOnAttribute</name>
<position x="-352.0" y="0.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Routing Strategy</name>
<value>Route to Property name</value>
</property>
<property>
<name>audit</name>
<value>${source_file:contains("audit")}</value>
</property>
</processor>
<processor>
<id>f92d3f77-958a-1344-bd3b-7c93457e5c12</id>
<name>Extract message</name> <name>Extract message</name>
<position x="352.0" y="280.0" /> <position x="-360.0" y="-216.0" />
<styles /> <styles />
<comment /> <comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class> <class>org.apache.nifi.processors.standard.ConvertRecord</class>
...@@ -6664,11 +6803,12 @@ ...@@ -6664,11 +6803,12 @@
<name>include-zero-record-flowfiles</name> <name>include-zero-record-flowfiles</name>
<value>true</value> <value>true</value>
</property> </property>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor> </processor>
<processor> <processor>
<id>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</id> <id>92693a34-99da-1004-adfb-bdf4aa7e1c30</id>
<name>Convert to json</name> <name>Convert to json</name>
<position x="1064.0" y="272.0" /> <position x="352.0" y="240.0" />
<styles /> <styles />
<comment /> <comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class> <class>org.apache.nifi.processors.standard.ConvertRecord</class>
...@@ -6689,7 +6829,7 @@ ...@@ -6689,7 +6829,7 @@
<runDurationNanos>0</runDurationNanos> <runDurationNanos>0</runDurationNanos>
<property> <property>
<name>record-reader</name> <name>record-reader</name>
<value>56ebe0aa-0176-1000-ffff-ffffbd212f01</value> <value>14453a95-7646-1485-0000-00002c675762</value>
</property> </property>
<property> <property>
<name>record-writer</name> <name>record-writer</name>
...@@ -6701,9 +6841,9 @@ ...@@ -6701,9 +6841,9 @@
</property> </property>
</processor> </processor>
<processor> <processor>
<id>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</id> <id>48723b8e-fae0-14e6-afdc-85c239646dc0</id>
<name>UpdateAttribute</name> <name>UpdateAttribute</name>
<position x="1072.0" y="472.0" /> <position x="-320.0" y="648.0" />
<styles /> <styles />
<comment /> <comment />
<class>org.apache.nifi.processors.attributes.UpdateAttribute</class> <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
...@@ -6738,66 +6878,101 @@ ...@@ -6738,66 +6878,101 @@
</property> </property>
<property> <property>
<name>data_index</name> <name>data_index</name>
<value>logs-haproxy</value> <value>logs-mysql</value>
</property> </property>
<property> <property>
<name>enrich_ip1</name> <name>enrich_ip1</name>
<value>/client.ip</value> <value>/client.ip</value>
</property> </property>
</processor> </processor>
<processor>
<id>14453a41-7646-1485-b398-28f819de4a45</id>
<name>Convert to json</name>
<position x="-336.0" y="200.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>70ea12d7-0176-1000-ffff-ffffee2ee306</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
<name>include-zero-record-flowfiles</name>
<value>false</value>
</property>
</processor>
<inputPort> <inputPort>
<id>65a33e05-e157-1bfc-8741-adf11b3df720</id> <id>7f683020-779c-1bc9-85da-5bad079d5d9d</id>
<name>Input</name> <name>Input</name>
<position x="397.9999517774115" y="110.99999315685733" /> <position x="-312.0" y="-336.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</inputPort> </inputPort>
<outputPort> <outputPort>
<id>328b35e2-eb52-1f47-b84d-52941eff8a07</id> <id>bcbb33ba-112e-1f53-8982-d5ae9f0e701f</id>
<name>Output</name> <name>Output</name>
<position x="1120.0" y="808.0" /> <position x="-256.0" y="960.0" />
<comments /> <comments />
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</outputPort> </outputPort>
<funnel> <funnel>
<id>312d3490-461e-13ac-a3a2-603704c456e2</id> <id>7113dbce-0176-1000-ffff-ffffbbfa695f</id>
<position x="8.0" y="424.0" /> <position x="-673.331668377643" y="376.49854987272295" />
</funnel> </funnel>
<funnel> <funnel>
<id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id> <id>f1b33d4c-1b10-18ad-ab4a-4a3a1e744f4b</id>
<position x="1882.9999517774115" y="327.9999931568573" /> <position x="1112.0" y="376.0" />
</funnel> </funnel>
<connection> <connection>
<id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id> <id>631e37d8-ca81-1bfa-8f55-aac2a22873ad</id>
<name /> <name />
<bendPoints /> <bendPoints />
<labelIndex>1</labelIndex> <labelIndex>1</labelIndex>
<zIndex>0</zIndex> <zIndex>0</zIndex>
<sourceId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</sourceId> <sourceId>50813f6b-a5f6-1a98-8ae4-115134714332</sourceId>
<sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType> <sourceType>PROCESSOR</sourceType>
<destinationId>328b35e2-eb52-1f47-b84d-52941eff8a07</destinationId> <destinationId>f1b33d4c-1b10-18ad-ab4a-4a3a1e744f4b</destinationId>
<destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType> <destinationType>FUNNEL</destinationType>
<relationship>success</relationship> <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration> <flowFileExpiration>60 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute /> <partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection> <connection>
<id>0ecb3e12-768e-1896-a850-2a2bec52eb95</id> <id>e43535a1-23e9-15af-9f98-2061dd6f97d6</id>
<name /> <name />
<bendPoints /> <bendPoints />
<labelIndex>1</labelIndex> <labelIndex>1</labelIndex>
<zIndex>0</zIndex> <zIndex>0</zIndex>
<sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId> <sourceId>92693a34-99da-1004-adfb-bdf4aa7e1c30</sourceId>
<sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType> <sourceType>PROCESSOR</sourceType>
<destinationId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</destinationId> <destinationId>50813f6b-a5f6-1a98-8ae4-115134714332</destinationId>
<destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType> <destinationType>PROCESSOR</destinationType>
<relationship>success</relationship> <relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueSize>10000</maxWorkQueueSize>
...@@ -6808,16 +6983,484 @@ ...@@ -6808,16 +6983,484 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection> <connection>
<id>9451307c-96df-1302-8189-8e556060bb80</id> <id>70e77065-0176-1000-0000-00001479fdf4</id>
<name /> <name />
<bendPoints /> <bendPoints />
<labelIndex>1</labelIndex> <labelIndex>1</labelIndex>
<zIndex>0</zIndex> <zIndex>0</zIndex>
<sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId> <sourceId>e0bd3907-2d13-1407-b2dd-48591e65e59d</sourceId>
<sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType> <sourceType>PROCESSOR</sourceType>
<destinationId>bb763b6c-302d-12a4-8eb2-b3b501d92244</destinationId> <destinationId>48723b8e-fae0-14e6-afdc-85c239646dc0</destinationId>
<destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>711609c1-0176-1000-0000-000013fdee3b</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>e0bd3907-2d13-1407-b2dd-48591e65e59d</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>7113dbce-0176-1000-ffff-ffffbbfa695f</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>FUNNEL</destinationType>
<relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>14453fcf-7646-1485-ffff-ffff952df142</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>e4353681-23e9-15af-0000-000032ea35e3</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>14453e90-7646-1485-ffff-ffff81f3c683</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>audit</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>7fe931b3-82b3-1699-b49a-d380dd14a5b8</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>48723b8e-fae0-14e6-afdc-85c239646dc0</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>bcbb33ba-112e-1f53-8982-d5ae9f0e701f</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>a35e3744-5906-1ee9-abc4-205356ca01d1</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>f92d3f77-958a-1344-bd3b-7c93457e5c12</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>e4353681-23e9-15af-0000-000032ea35e3</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>70e8f3cb-0176-1000-0000-00006d2cdbf5</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>14453a41-7646-1485-b398-28f819de4a45</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>e0bd3907-2d13-1407-b2dd-48591e65e59d</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>14453eaa-7646-1485-0000-000070b97065</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>14453e90-7646-1485-ffff-ffff81f3c683</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>92693a34-99da-1004-adfb-bdf4aa7e1c30</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>cf95350a-de6c-1a4b-8183-8f9cfa11449a</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>7f683020-779c-1bc9-85da-5bad079d5d9d</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>f92d3f77-958a-1344-bd3b-7c93457e5c12</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>3e21311d-dc5c-143f-b39e-d8fb8c9fd36d</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>50813f6b-a5f6-1a98-8ae4-115134714332</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>48723b8e-fae0-14e6-afdc-85c239646dc0</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>873a3c42-fe81-1ce1-8c70-7da1f2542c31</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>92693a34-99da-1004-adfb-bdf4aa7e1c30</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>f1b33d4c-1b10-18ad-ab4a-4a3a1e744f4b</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>FUNNEL</destinationType>
<relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>14453a4b-7646-1485-ffff-fffffc8f5285</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>e4353681-23e9-15af-0000-000032ea35e3</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>14453a41-7646-1485-b398-28f819de4a45</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>unmatched</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>71140678-0176-1000-0000-000060437da4</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>14453a41-7646-1485-b398-28f819de4a45</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>7113dbce-0176-1000-ffff-ffffbbfa695f</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>FUNNEL</destinationType>
<relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
<processGroup>
<id>5d04357e-423c-1ab5-a7a4-44565abfed7f</id>
<name>Haproxy</name>
<position x="-448.0" y="664.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>c9763c4c-7186-1460-871a-b5fd00ca3241</id>
<name>UpdateRecord</name>
<position x="352.0" y="472.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
<name>replacement-value-strategy</name>
<value>literal-value</value>
</property>
<property>
<name>/labels/source_host</name>
<value>${source_host}</value>
</property>
<property>
<name>/timestamp</name>
<value>${field.value:toDate('dd/MMM/yyyy:HH:mm:ss.SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
</property>
</processor>
<processor>
<id>e4c8356d-54ad-15b5-94fe-799d9465aa51</id>
<name>Extract message</name>
<position x="352.0" y="280.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
<value>bc8e5957-0175-1000-0000-00003346421d</value>
</property>
<property>
<name>include-zero-record-flowfiles</name>
<value>true</value>
</property>
</processor>
<processor>
<id>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</id>
<name>Convert to json</name>
<position x="1064.0" y="272.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ConvertRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
<value>56ebe0aa-0176-1000-ffff-ffffbd212f01</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
<name>include-zero-record-flowfiles</name>
<value>false</value>
</property>
</processor>
<processor>
<id>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</id>
<name>UpdateAttribute</name>
<position x="1072.0" y="472.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-update-attribute-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>Delete Attributes Expression</name>
</property>
<property>
<name>Store State</name>
<value>Do not store state</value>
</property>
<property>
<name>Stateful Variables Initial Value</name>
</property>
<property>
<name>canonical-value-lookup-cache-size</name>
<value>100</value>
</property>
<property>
<name>data_index</name>
<value>logs-haproxy</value>
</property>
<property>
<name>enrich_ip1</name>
<value>/client.ip</value>
</property>
</processor>
<inputPort>
<id>65a33e05-e157-1bfc-8741-adf11b3df720</id>
<name>Input</name>
<position x="397.9999517774115" y="110.99999315685733" />
<comments />
<scheduledState>RUNNING</scheduledState>
</inputPort>
<outputPort>
<id>328b35e2-eb52-1f47-b84d-52941eff8a07</id>
<name>Output</name>
<position x="1120.0" y="808.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<funnel>
<id>312d3490-461e-13ac-a3a2-603704c456e2</id>
<position x="8.0" y="424.0" />
</funnel>
<funnel>
<id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id>
<position x="1882.9999517774115" y="327.9999931568573" />
</funnel>
<connection>
<id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</sourceId>
<sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>328b35e2-eb52-1f47-b84d-52941eff8a07</destinationId>
<destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>0ecb3e12-768e-1896-a850-2a2bec52eb95</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId>
<sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</destinationId>
<destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>9451307c-96df-1302-8189-8e556060bb80</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId>
<sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>bb763b6c-302d-12a4-8eb2-b3b501d92244</destinationId>
<destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
<destinationType>FUNNEL</destinationType> <destinationType>FUNNEL</destinationType>
<relationship>failure</relationship> <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueSize>10000</maxWorkQueueSize>
...@@ -7600,6 +8243,26 @@ ...@@ -7600,6 +8243,26 @@
<partitioningAttribute /> <partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection>
<id>e43535d0-23e9-15af-ffff-ffffa44d6172</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>bcbb33ba-112e-1f53-8982-d5ae9f0e701f</sourceId>
<sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
<destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection> <connection>
<id>349b339b-a821-1197-0000-00002e648df6</id> <id>349b339b-a821-1197-0000-00002e648df6</id>
<name /> <name />
...@@ -7867,6 +8530,29 @@ ...@@ -7867,6 +8530,29 @@
<partitioningAttribute /> <partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection>
<id>e43535c9-23e9-15af-ffff-ffffcd7d888a</id>
<name />
<bendPoints>
<bendPoint x="-704.0" y="1256.0" />
<bendPoint x="-584.0" y="1368.0" />
</bendPoints>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
<sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>7f683020-779c-1bc9-85da-5bad079d5d9d</destinationId>
<destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<relationship>mysql</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup> </processGroup>
<processGroup> <processGroup>
<id>b3d57504-7c06-37a3-b59b-8723f60fa728</id> <id>b3d57504-7c06-37a3-b59b-8723f60fa728</id>
...@@ -8380,7 +9066,7 @@ ...@@ -8380,7 +9066,7 @@
</property> </property>
<property> <property>
<name>Password</name> <name>Password</name>
<value>enc{7f706f76bc019ad8a3c7700ec14f6d65035b47cfa70fce4d0aade0809ded55af0afc391ccf1744443ffa082fc97204a6}</value> <value>enc{1c4c58f74f95b67649d56061742c742acc78f69ccc57c9e9ef50d02d079f56126e9174e3f3651d3f58f0fd58976866f6}</value>
</property> </property>
<property> <property>
<name>elasticsearch-http-connect-timeout</name> <name>elasticsearch-http-connect-timeout</name>
...@@ -11031,8 +11717,245 @@ ...@@ -11031,8 +11717,245 @@
</processGroup> </processGroup>
</processGroup> </processGroup>
<controllerService> <controllerService>
<id>bc97858d-0175-1000-0000-0000130a84f8</id> <id>349b34c7-a821-1197-ffff-ffff85d82877</id>
<name>Nifi logs GrokReader</name> <name>Contry code to region</name>
<comment />
<class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>csv-file</name>
<value>/opt/nifi/nifi-current/conf/enrich/CountriesWithRegionalCodes.csv</value>
</property>
<property>
<name>CSV Format</name>
<value>default</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>lookup-key-column</name>
<value>alpha-2</value>
</property>
<property>
<name>ignore-duplicates</name>
<value>true</value>
</property>
<property>
<name>Value Separator</name>
<value>,</value>
</property>
<property>
<name>Quote Character</name>
<value>"</value>
</property>
<property>
<name>Quote Mode</name>
<value>MINIMAL</value>
</property>
<property>
<name>Comment Marker</name>
</property>
<property>
<name>Escape Character</name>
<value>\</value>
</property>
<property>
<name>Trim Fields</name>
<value>true</value>
</property>
<property>
<name>lookup-value-column</name>
<value>region</value>
</property>
</controllerService>
<controllerService>
<id>bbd4d3a2-0175-1000-0000-00000b0fb8bd</id>
<name>Tor node CSV</name>
<comment />
<class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>csv-file</name>
<value>/opt/nifi/nifi-current/conf/enrich/tornodes.csv</value>
</property>
<property>
<name>CSV Format</name>
<value>default</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>lookup-key-column</name>
<value>ip_addr</value>
</property>
<property>
<name>ignore-duplicates</name>
<value>true</value>
</property>
<property>
<name>Value Separator</name>
<value>,</value>
</property>
<property>
<name>Quote Character</name>
<value>"</value>
</property>
<property>
<name>Quote Mode</name>
<value>MINIMAL</value>
</property>
<property>
<name>Comment Marker</name>
</property>
<property>
<name>Escape Character</name>
<value>\</value>
</property>
<property>
<name>Trim Fields</name>
<value>true</value>
</property>
<property>
<name>lookup-value-column</name>
<value>ip_addr</value>
</property>
</controllerService>
<controllerService>
<id>14453a95-7646-1485-0000-00002c675762</id>
<name>Mysql audit log</name>
<comment />
<class>org.apache.nifi.csv.CSVReader</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>schema-access-strategy</name>
<value>infer-schema</value>
</property>
<property>
<name>schema-registry</name>
</property>
<property>
<name>schema-name</name>
<value>${schema.name}</value>
</property>
<property>
<name>schema-version</name>
</property>
<property>
<name>schema-branch</name>
</property>
<property>
<name>schema-text</name>
<value>${avro.schema}</value>
</property>
<property>
<name>csv-reader-csv-parser</name>
<value>commons-csv</value>
</property>
<property>
<name>Date Format</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
</property>
<property>
<name>CSV Format</name>
<value>custom</value>
</property>
<property>
<name>Value Separator</name>
<value>,</value>
</property>
<property>
<name>Record Separator</name>
<value>\n</value>
</property>
<property>
<name>Skip Header Line</name>
<value>true</value>
</property>
<property>
<name>ignore-csv-header</name>
<value>false</value>
</property>
<property>
<name>Quote Character</name>
<value>"</value>
</property>
<property>
<name>Escape Character</name>
<value>\</value>
</property>
<property>
<name>Comment Marker</name>
</property>
<property>
<name>Null String</name>
</property>
<property>
<name>Trim Fields</name>
<value>true</value>
</property>
<property>
<name>csvutils-character-set</name>
<value>UTF-8</value>
</property>
</controllerService>
<controllerService>
<id>8b1dd8bb-0170-1000-0000-000007446e6a</id>
<name>Misp DistributedMapCacheServer</name>
<comment />
<class>org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-distributed-cache-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Port</name>
<value>4557</value>
</property>
<property>
<name>Maximum Cache Entries</name>
<value>10000</value>
</property>
<property>
<name>Eviction Strategy</name>
<value>Least Frequently Used</value>
</property>
<property>
<name>Persistence Directory</name>
<value>/opt/nifi/nifi-current/conf/</value>
</property>
<property>
<name>SSL Context Service</name>
</property>
</controllerService>
<controllerService>
<id>56ebe0aa-0176-1000-ffff-ffffbd212f01</id>
<name>Haproxy GrokReader</name>
<comment /> <comment />
<class>org.apache.nifi.grok.GrokReader</class> <class>org.apache.nifi.grok.GrokReader</class>
<bundle> <bundle>
...@@ -11064,10 +11987,11 @@ ...@@ -11064,10 +11987,11 @@
</property> </property>
<property> <property>
<name>Grok Pattern File</name> <name>Grok Pattern File</name>
<value>/opt/nifi/nifi-current/conf/enrich/haproxy.groklib</value>
</property> </property>
<property> <property>
<name>Grok Expression</name> <name>Grok Expression</name>
<value>%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:class} %{GREEDYDATA:message}</value> <value>%{PROG:process.name}(?:\[%{POSINT:process.pid}\])?: %{HAPROXYHTTPBASE}</value>
</property> </property>
<property> <property>
<name>no-match-behavior</name> <name>no-match-behavior</name>
...@@ -11075,62 +11999,47 @@ ...@@ -11075,62 +11999,47 @@
</property> </property>
</controllerService> </controllerService>
<controllerService> <controllerService>
<id>349b34c7-a821-1197-ffff-ffff85d82877</id> <id>bc97858d-0175-1000-0000-0000130a84f8</id>
<name>Contry code to region</name> <name>Nifi logs GrokReader</name>
<comment /> <comment />
<class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class> <class>org.apache.nifi.grok.GrokReader</class>
<bundle> <bundle>
<group>org.apache.nifi</group> <group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact> <artifact>nifi-record-serialization-services-nar</artifact>
<version>1.12.1</version> <version>1.12.1</version>
</bundle> </bundle>
<enabled>true</enabled> <enabled>true</enabled>
<property> <property>
<name>csv-file</name> <name>schema-access-strategy</name>
<value>/opt/nifi/nifi-current/conf/enrich/CountriesWithRegionalCodes.csv</value> <value>string-fields-from-grok-expression</value>
</property>
<property>
<name>CSV Format</name>
<value>default</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>lookup-key-column</name>
<value>alpha-2</value>
</property> </property>
<property> <property>
<name>ignore-duplicates</name> <name>schema-registry</name>
<value>true</value>
</property> </property>
<property> <property>
<name>Value Separator</name> <name>schema-name</name>
<value>,</value> <value>${schema.name}</value>
</property> </property>
<property> <property>
<name>Quote Character</name> <name>schema-version</name>
<value>"</value>
</property> </property>
<property> <property>
<name>Quote Mode</name> <name>schema-branch</name>
<value>MINIMAL</value>
</property> </property>
<property> <property>
<name>Comment Marker</name> <name>schema-text</name>
<value>${avro.schema}</value>
</property> </property>
<property> <property>
<name>Escape Character</name> <name>Grok Pattern File</name>
<value>\</value>
</property> </property>
<property> <property>
<name>Trim Fields</name> <name>Grok Expression</name>
<value>true</value> <value>%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:class} %{GREEDYDATA:message}</value>
</property> </property>
<property> <property>
<name>lookup-value-column</name> <name>no-match-behavior</name>
<value>region</value> <value>append-to-previous-message</value>
</property> </property>
</controllerService> </controllerService>
<controllerService> <controllerService>
...@@ -11153,65 +12062,6 @@ ...@@ -11153,65 +12062,6 @@
<value>UTF-8</value> <value>UTF-8</value>
</property> </property>
</controllerService> </controllerService>
<controllerService>
<id>bbd4d3a2-0175-1000-0000-00000b0fb8bd</id>
<name>Tor node CSV</name>
<comment />
<class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>csv-file</name>
<value>/opt/nifi/nifi-current/conf/enrich/tornodes.csv</value>
</property>
<property>
<name>CSV Format</name>
<value>default</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>lookup-key-column</name>
<value>ip_addr</value>
</property>
<property>
<name>ignore-duplicates</name>
<value>true</value>
</property>
<property>
<name>Value Separator</name>
<value>,</value>
</property>
<property>
<name>Quote Character</name>
<value>"</value>
</property>
<property>
<name>Quote Mode</name>
<value>MINIMAL</value>
</property>
<property>
<name>Comment Marker</name>
</property>
<property>
<name>Escape Character</name>
<value>\</value>
</property>
<property>
<name>Trim Fields</name>
<value>true</value>
</property>
<property>
<name>lookup-value-column</name>
<value>ip_addr</value>
</property>
</controllerService>
<controllerService> <controllerService>
<id>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</id> <id>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</id>
<name>Misp Lookup Service</name> <name>Misp Lookup Service</name>
...@@ -11259,6 +12109,46 @@ ...@@ -11259,6 +12109,46 @@
<value>30 secs</value> <value>30 secs</value>
</property> </property>
</controllerService> </controllerService>
<controllerService>
<id>83443c00-b286-366a-b8e0-2f51527ab8e5</id>
<name>Common CA</name>
<comment />
<class>org.apache.nifi.ssl.StandardRestrictedSSLContextService</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-ssl-context-service-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Keystore Filename</name>
</property>
<property>
<name>Keystore Password</name>
</property>
<property>
<name>key-password</name>
</property>
<property>
<name>Keystore Type</name>
</property>
<property>
<name>Truststore Filename</name>
<value>/opt/nifi/nifi-current/conf/cacerts.jks</value>
</property>
<property>
<name>Truststore Password</name>
<value>enc{9ca8dddd769be18fff9e22016c86d9f05a3a987f834c79a92609b7faa5e1e840}</value>
</property>
<property>
<name>Truststore Type</name>
<value>JKS</value>
</property>
<property>
<name>SSL Protocol</name>
<value>TLS</value>
</property>
</controllerService>
<controllerService> <controllerService>
<id>17b30955-5464-3709-8a32-69a459850cfa</id> <id>17b30955-5464-3709-8a32-69a459850cfa</id>
<name>Inferred JsonRecordSetWriter</name> <name>Inferred JsonRecordSetWriter</name>
...@@ -11333,79 +12223,8 @@ ...@@ -11333,79 +12223,8 @@
</property> </property>
</controllerService> </controllerService>
<controllerService> <controllerService>
<id>8b1dd8bb-0170-1000-0000-000007446e6a</id> <id>70ea12d7-0176-1000-ffff-ffffee2ee306</id>
<name>Misp DistributedMapCacheServer</name> <name>Mysql log GrokReader</name>
<comment />
<class>org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-distributed-cache-services-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Port</name>
<value>4557</value>
</property>
<property>
<name>Maximum Cache Entries</name>
<value>10000</value>
</property>
<property>
<name>Eviction Strategy</name>
<value>Least Frequently Used</value>
</property>
<property>
<name>Persistence Directory</name>
<value>/opt/nifi/nifi-current/conf/</value>
</property>
<property>
<name>SSL Context Service</name>
</property>
</controllerService>
<controllerService>
<id>83443c00-b286-366a-b8e0-2f51527ab8e5</id>
<name>Common CA</name>
<comment />
<class>org.apache.nifi.ssl.StandardRestrictedSSLContextService</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-ssl-context-service-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>Keystore Filename</name>
</property>
<property>
<name>Keystore Password</name>
</property>
<property>
<name>key-password</name>
</property>
<property>
<name>Keystore Type</name>
</property>
<property>
<name>Truststore Filename</name>
<value>/opt/nifi/nifi-current/conf/cacerts.jks</value>
</property>
<property>
<name>Truststore Password</name>
<value>enc{fb94647e0f686a70205e73bdc73eb6a28bdbcd74c3e169b4cd116dd6a7cc28f6}</value>
</property>
<property>
<name>Truststore Type</name>
<value>JKS</value>
</property>
<property>
<name>SSL Protocol</name>
<value>TLS</value>
</property>
</controllerService>
<controllerService>
<id>56ebe0aa-0176-1000-ffff-ffffbd212f01</id>
<name>Haproxy GrokReader</name>
<comment /> <comment />
<class>org.apache.nifi.grok.GrokReader</class> <class>org.apache.nifi.grok.GrokReader</class>
<bundle> <bundle>
...@@ -11437,11 +12256,10 @@ ...@@ -11437,11 +12256,10 @@
</property> </property>
<property> <property>
<name>Grok Pattern File</name> <name>Grok Pattern File</name>
<value>/opt/nifi/nifi-current/conf/enrich/haproxy.groklib</value>
</property> </property>
<property> <property>
<name>Grok Expression</name> <name>Grok Expression</name>
<value>%{PROG:process.name}(?:\[%{POSINT:process.pid}\])?: %{HAPROXYHTTPBASE}</value> <value>%{GREEDYDATA:timestamp} %{DATA:process}: %{GREEDYDATA:message}</value>
</property> </property>
<property> <property>
<name>no-match-behavior</name> <name>no-match-behavior</name>
... ...
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment