From 6badb5049e4ebcd6c331f80f0122cef7d5bb424e Mon Sep 17 00:00:00 2001
From: Arne Oslebo <arne.oslebo@uninett.no>
Date: Sat, 12 Dec 2020 14:32:28 +0100
Subject: [PATCH] added parsing of haproxy logs to nifi
---
inventories/filebeat | 1 +
roles/haproxy/templates/haproxy.cfg.j2 | 9 +
roles/nifi/files/haproxy.groklib | 39 ++
roles/nifi/tasks/main.yml | 7 +
roles/nifi/templates/flow.xml.j2 | 523 ++++++++++++++++++++++--
roles/nifi/templates/nifi.properties.j2 | 4 +-
6 files changed, 537 insertions(+), 46 deletions(-)
create mode 100644 roles/nifi/files/haproxy.groklib
diff --git a/inventories/filebeat b/inventories/filebeat
index 9f4153a..e8df75e 100644
--- a/inventories/filebeat
+++ b/inventories/filebeat
@@ -13,3 +13,4 @@ soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervis
soctools-cortex ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cortex" FILEBEAT_LOG_FORMAT="text"
soctools-thehive ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="thehive" FILEBEAT_LOG_FORMAT="text"
soctools-cassandra ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cassandra" FILEBEAT_LOG_FORMAT="text"
+
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2
index d3c0eef..f35e48e 100644
--- a/roles/haproxy/templates/haproxy.cfg.j2
+++ b/roles/haproxy/templates/haproxy.cfg.j2
@@ -31,6 +31,7 @@ listen nifiserv
fullconn 5000
balance source
option tcpka
+ option httplog
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:9443 ssl check verify none
{% endfor %}
@@ -42,6 +43,7 @@ listen odfeserv
fullconn 5000
balance source
option tcpka
+ option httplog
{% for odfehost in groups['odfeescontainers'] %}
server {{odfehost}} {{odfehost}}:9200 ssl check verify none
{% endfor %}
@@ -53,6 +55,7 @@ listen keycloakserv
fullconn 5000
balance source
option tcpka
+ option httplog
{% for keycloakhost in groups['keycloakcontainers'] %}
server {{keycloakhost}} {{keycloakhost}}:8443 ssl check verify none
{% endfor %}
@@ -65,6 +68,7 @@ listen thehiveserv
balance source
option tcpka
option forwardfor
+ option httplog
{% for thehivehost in groups['thehive'] %}
server {{thehivehost}} {{thehivehost}}:9000 check verify none
{% endfor %}
@@ -77,6 +81,7 @@ listen cortexserv
balance source
option tcpka
option forwardfor
+ option httplog
{% for cortexhost in groups['cortex'] %}
server {{cortexhost}} {{cortexhost}}:9001 check verify none
{% endfor %}
@@ -90,6 +95,7 @@ listen nifiservtcp77{{port}}
balance source
option tcpka
option tcp-check
+ option tcplog
tcp-check connect port 77{{port}}
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:77{{port}} check
@@ -106,6 +112,7 @@ listen nifiservtcp7771
balance source
option tcpka
option tcp-check
+ option tcplog
tcp-check connect port 7771
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:7771 check
@@ -120,6 +127,7 @@ listen nifiservhttp{{port}}
fullconn 5000
balance source
option tcpka
+ option httplog
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:{{port}} check
{% endfor %}
@@ -136,6 +144,7 @@ listen nifiservtcp{{port}}
balance source
option tcpka
option tcp-check
+ option tcplog
tcp-check connect port {{port}}
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:{{port}} check
diff --git a/roles/nifi/files/haproxy.groklib b/roles/nifi/files/haproxy.groklib
new file mode 100644
index 0000000..37517af
--- /dev/null
+++ b/roles/nifi/files/haproxy.groklib
@@ -0,0 +1,39 @@
+## These patterns were tested w/ haproxy-1.4.15
+
+## Documentation of the haproxy log formats can be found at the following links:
+## http://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat
+## http://code.google.com/p/haproxy-docs/wiki/TCPLogFormat
+
+HAPROXYTIME (?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9])
+HAPROXYDATE %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds}
+
+# Override these default patterns to parse out what is captured in your haproxy.cfg
+HAPROXYCAPTUREDREQUESTHEADERS %{DATA:captured_request_headers}
+HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers}
+
+# Example:
+# These haproxy config lines will add data to the logs that are captured
+# by the patterns below. Place them in your custom patterns directory to
+# override the defaults.
+#
+# capture request header Host len 40
+# capture request header X-Forwarded-For len 50
+# capture request header Accept-Language len 50
+# capture request header Referer len 200
+# capture request header User-Agent len 200
+#
+# capture response header Content-Type len 30
+# capture response header Content-Encoding len 10
+# capture response header Cache-Control len 200
+# capture response header Last-Modified len 200
+#
+# HAPROXYCAPTUREDREQUESTHEADERS %{DATA:request_header_host}\|%{DATA:request_header_x_forwarded_for}\|%{DATA:request_header_accept_language}\|%{DATA:request_header_referer}\|%{DATA:request_header_user_agent}
+# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified}
+
+# parse a haproxy 'httplog' line
+HAPROXYHTTPBASE %{IP:client.ip}:%{INT:client.port} \[%{HAPROXYDATE:timestamp}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server.domain} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
+
+HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
+
+# parse a haproxy 'tcplog' line
+HAPROXYTCP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client.ip}:%{INT:client.port} \[%{HAPROXYDATE:timestamp}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue}
\ No newline at end of file
diff --git a/roles/nifi/tasks/main.yml b/roles/nifi/tasks/main.yml
index 392a6dd..30cb34c 100644
--- a/roles/nifi/tasks/main.yml
+++ b/roles/nifi/tasks/main.yml
@@ -129,6 +129,13 @@
tags:
- start
+- name: Copy grok libraries
+ copy:
+ src: haproxy.groklib
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib"
+ tags:
+ - start
+
- name: Create empty list of Tor nodes
remote_user: nifi
copy:
diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2
index 8fbd37f..8e8cc20 100644
--- a/roles/nifi/templates/flow.xml.j2
+++ b/roles/nifi/templates/flow.xml.j2
@@ -2759,8 +2759,8 @@
<value>/opt/nifi/nifi-current/conf/enrich/freq</value>
</property>
<property>
- <name>Frequency File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
@@ -2771,16 +2771,16 @@
<value>${enrich_domain1}</value>
</property>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>Result Field 1</name>
+ <value>${enrich_domain1}_freq1</value>
</property>
<property>
<name>Result Field 2</name>
<value>${enrich_domain1}_freq2</value>
</property>
<property>
- <name>Result Field 1</name>
- <value>${enrich_domain1}_freq1</value>
+ <name>Frequency File</name>
+ <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value>
</property>
</processor>
<inputPort>
@@ -3432,8 +3432,8 @@
<value>/opt/nifi/nifi-current/conf/enrich/freq</value>
</property>
<property>
- <name>Frequency File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
@@ -3444,16 +3444,16 @@
<value>${enrich_domain2}</value>
</property>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>Result Field 1</name>
+ <value>${enrich_domain2}_freq1</value>
</property>
<property>
<name>Result Field 2</name>
<value>${enrich_domain2}_freq2</value>
</property>
<property>
- <name>Result Field 1</name>
- <value>${enrich_domain2}_freq1</value>
+ <name>Frequency File</name>
+ <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value>
</property>
</processor>
<inputPort>
@@ -4205,16 +4205,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
- <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
- <name>To enrichment</name>
- <position x="480.0" y="392.0" />
+ <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
+ <name>To data output</name>
+ <position x="-1120.0" y="592.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<outputPort>
- <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
- <name>To data output</name>
- <position x="-1120.0" y="592.0" />
+ <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
+ <name>To enrichment</name>
+ <position x="480.0" y="392.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -4226,16 +4226,16 @@
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
- <id>27d5761b-0172-1000-0000-000059275dad</id>
- <name>To enrichment</name>
- <position x="-312.0" y="328.0" />
+ <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
+ <name>To data output</name>
+ <position x="-632.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
<outputPort>
- <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
- <name>To data output</name>
- <position x="-632.0" y="328.0" />
+ <id>27d5761b-0172-1000-0000-000059275dad</id>
+ <name>To enrichment</name>
+ <position x="-312.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
@@ -4277,6 +4277,10 @@
<name>suricata</name>
<value>${log_type:equals("suricata")}</value>
</property>
+ <property>
+ <name>haproxy</name>
+ <value>${log_type:equals("haproxy")}</value>
+ </property>
<property>
<name>zeek</name>
<value>${log_type:equals("zeek")}</value>
@@ -4287,16 +4291,16 @@
</property>
</processor>
<outputPort>
- <id>bcb879d5-0175-1000-0000-000070879ad0</id>
- <name>To data output</name>
- <position x="-2480.0" y="336.0" />
+ <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
+ <name>To enrichment</name>
+ <position x="360.0" y="424.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<outputPort>
- <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
- <name>To enrichment</name>
- <position x="352.0" y="296.0" />
+ <id>bcb879d5-0175-1000-0000-000070879ad0</id>
+ <name>To data output</name>
+ <position x="-2480.0" y="336.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -4582,14 +4586,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>bc925474-0175-1000-0000-00004e78071f</id>
- <position x="1882.9999517774115" y="327.9999931568573" />
- </funnel>
<funnel>
<id>bc90d189-0175-1000-0000-0000037bc986</id>
<position x="8.0" y="424.0" />
</funnel>
+ <funnel>
+ <id>bc925474-0175-1000-0000-00004e78071f</id>
+ <position x="1882.9999517774115" y="327.9999931568573" />
+ </funnel>
<connection>
<id>bc90c7ac-0175-1000-ffff-fffffa80b534</id>
<name />
@@ -5096,14 +5100,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
- <position x="-1446.1517058240609" y="301.4492766741185" />
- </funnel>
<funnel>
<id>895faa7a-0175-1000-0000-000014ef9dd3</id>
<position x="278.84829417593915" y="332.4492766741185" />
</funnel>
+ <funnel>
+ <id>895f7db3-0175-1000-ffff-ffff8229d688</id>
+ <position x="-1446.1517058240609" y="301.4492766741185" />
+ </funnel>
<connection>
<id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
<name />
@@ -6568,6 +6572,350 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
+ <processGroup>
+ <id>5d04357e-423c-1ab5-a7a4-44565abfed7f</id>
+ <name>Haproxy</name>
+ <position x="-448.0" y="664.0" />
+ <comment />
+ <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
+ <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
+ <processor>
+ <id>c9763c4c-7186-1460-871a-b5fd00ca3241</id>
+ <name>UpdateRecord</name>
+ <position x="352.0" y="472.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.UpdateRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ </property>
+ <property>
+ <name>replacement-value-strategy</name>
+ <value>literal-value</value>
+ </property>
+ <property>
+ <name>/labels/source_host</name>
+ <value>${source_host}</value>
+ </property>
+ <property>
+ <name>/timestamp</name>
+ <value>${field.value:toDate('dd/MMM/yyyy:HH:mm:ss.SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
+ </property>
+ </processor>
+ <processor>
+ <id>e4c8356d-54ad-15b5-94fe-799d9465aa51</id>
+ <name>Extract message</name>
+ <position x="352.0" y="280.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ConvertRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>bc8e5957-0175-1000-0000-00003346421d</value>
+ </property>
+ <property>
+ <name>include-zero-record-flowfiles</name>
+ <value>true</value>
+ </property>
+ </processor>
+ <processor>
+ <id>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</id>
+ <name>Convert to json</name>
+ <position x="1064.0" y="272.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ConvertRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>56ebe0aa-0176-1000-ffff-ffffbd212f01</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ </property>
+ <property>
+ <name>include-zero-record-flowfiles</name>
+ <value>false</value>
+ </property>
+ </processor>
+ <processor>
+ <id>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</id>
+ <name>UpdateAttribute</name>
+ <position x="1072.0" y="472.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-haproxy</value>
+ </property>
+ </processor>
+ <inputPort>
+ <id>65a33e05-e157-1bfc-8741-adf11b3df720</id>
+ <name>Input</name>
+ <position x="397.9999517774115" y="110.99999315685733" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </inputPort>
+ <outputPort>
+ <id>328b35e2-eb52-1f47-b84d-52941eff8a07</id>
+ <name>Output</name>
+ <position x="1120.0" y="808.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </outputPort>
+ <funnel>
+ <id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id>
+ <position x="1882.9999517774115" y="327.9999931568573" />
+ </funnel>
+ <funnel>
+ <id>312d3490-461e-13ac-a3a2-603704c456e2</id>
+ <position x="8.0" y="424.0" />
+ </funnel>
+ <connection>
+ <id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>328b35e2-eb52-1f47-b84d-52941eff8a07</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>0ecb3e12-768e-1896-a850-2a2bec52eb95</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>9451307c-96df-1302-8189-8e556060bb80</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>bb763b6c-302d-12a4-8eb2-b3b501d92244</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>fbd23518-c750-15f2-b8e0-9a0600eb3246</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e4c8356d-54ad-15b5-94fe-799d9465aa51</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>312d3490-461e-13ac-a3a2-603704c456e2</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>60 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>b5d43cea-5555-10b0-b75f-b88a95e9c6aa</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>65a33e05-e157-1bfc-8741-adf11b3df720</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>e4c8356d-54ad-15b5-94fe-799d9465aa51</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>1285380e-3d8e-199c-a6b2-fec085b63cd4</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>312d3490-461e-13ac-a3a2-603704c456e2</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>60 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>484a3eab-4af3-11cd-abe2-d5ee6fc1a291</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e4c8356d-54ad-15b5-94fe-799d9465aa51</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>39ef3a2d-874e-11a6-87be-0b3582fa43de</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c9763c4c-7186-1460-871a-b5fd00ca3241</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
<processGroup>
<id>83691174-683f-3c7c-8526-8fc00397aee1</id>
<name>Zeek</name>
@@ -6784,14 +7132,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <funnel>
- <id>06521038-335b-3139-839d-ab43a013ce03</id>
- <position x="-1557.869726298236" y="758.8984861527665" />
- </funnel>
<funnel>
<id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
<position x="248.5321508445502" y="703.4412774751572" />
</funnel>
+ <funnel>
+ <id>06521038-335b-3139-839d-ab43a013ce03</id>
+ <position x="-1557.869726298236" y="758.8984861527665" />
+ </funnel>
<connection>
<id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
<name />
@@ -6973,6 +7321,48 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>56e5add3-0176-1000-ffff-ffffd667d1f6</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-584.0" y="624.0" />
+ </bendPoints>
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
+ <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>65a33e05-e157-1bfc-8741-adf11b3df720</destinationId>
+ <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship>haproxy</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId>
+ <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
+ <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>89630460-0175-1000-0000-00006b5f18c8</id>
<name />
@@ -7606,7 +7996,7 @@
</property>
<property>
<name>Password</name>
- <value>enc{a2a2e69fa7a4563faa60a9e069ae100df4117e257dfa1cdbe21ed388bc4480066d147f741a89f88b121b598a508212f6}</value>
+ <value>enc{1c9a67efa861b9a5f0ced47e1bb930650b19b788b8576e55d87fa2a3a4760d790d7425f299ed70ea1859a64a26753959}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
@@ -10618,7 +11008,7 @@
</property>
<property>
<name>Truststore Password</name>
- <value>enc{03ca02479138f0ae3484435afa13a8d10bdc66b2b9391e10764a3f022ad1f693}</value>
+ <value>enc{d064a1e3a5a974d37b0202bbb9551137b9543af176d965ad630f0fc2bdafa690}</value>
</property>
<property>
<name>Truststore Type</name>
@@ -10629,6 +11019,51 @@
<value>TLS</value>
</property>
</controllerService>
+ <controllerService>
+ <id>56ebe0aa-0176-1000-ffff-ffffbd212f01</id>
+ <name>Haproxy GrokReader</name>
+ <comment />
+ <class>org.apache.nifi.grok.GrokReader</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-record-serialization-services-nar</artifact>
+ <version>1.12.1</version>
+ </bundle>
+ <enabled>true</enabled>
+ <property>
+ <name>schema-access-strategy</name>
+ <value>string-fields-from-grok-expression</value>
+ </property>
+ <property>
+ <name>schema-registry</name>
+ </property>
+ <property>
+ <name>schema-name</name>
+ <value>${schema.name}</value>
+ </property>
+ <property>
+ <name>schema-version</name>
+ </property>
+ <property>
+ <name>schema-branch</name>
+ </property>
+ <property>
+ <name>schema-text</name>
+ <value>${avro.schema}</value>
+ </property>
+ <property>
+ <name>Grok Pattern File</name>
+ <value>/opt/nifi/nifi-current/conf/enrich/haproxy.groklib</value>
+ </property>
+ <property>
+ <name>Grok Expression</name>
+ <value>%{PROG:process.name}(?:\[%{POSINT:process.pid}\])?: %{HAPROXYHTTPBASE}</value>
+ </property>
+ <property>
+ <name>no-match-behavior</name>
+ <value>append-to-previous-message</value>
+ </property>
+ </controllerService>
<controllerService>
<id>179dd31f-89ed-3179-adb2-85a9c61869ce</id>
<name>Inferred JsonTreeReader</name>
diff --git a/roles/nifi/templates/nifi.properties.j2 b/roles/nifi/templates/nifi.properties.j2
index 6efbc1e..188f234 100644
--- a/roles/nifi/templates/nifi.properties.j2
+++ b/roles/nifi/templates/nifi.properties.j2
@@ -120,8 +120,8 @@ nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
-nifi.components.status.repository.buffer.size=1440
-nifi.components.status.snapshot.frequency=1 min
+nifi.components.status.repository.buffer.size=288
+nifi.components.status.snapshot.frequency=5 min
# Site to Site properties
nifi.remote.input.host={{ inventory_hostname }}
--
GitLab