added parsing of haproxy logs to nifi

inventories/filebeat

@@ -13,3 +13,4 @@ soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervis
 soctools-cortex ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cortex" FILEBEAT_LOG_FORMAT="text"
 soctools-thehive ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="thehive" FILEBEAT_LOG_FORMAT="text"
 soctools-cassandra ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cassandra" FILEBEAT_LOG_FORMAT="text"
+

roles/haproxy/templates/haproxy.cfg.j2

@@ -31,6 +31,7 @@ listen nifiserv
 	fullconn 5000
 	balance source
 	option tcpka
+        option httplog
 {% for nifihost in groups['nificontainers'] %}
 	server {{nifihost}} {{nifihost}}:9443 ssl check verify none
 {% endfor %}
@@ -42,6 +43,7 @@ listen odfeserv
         fullconn 5000
         balance source
         option tcpka
+        option httplog
 {% for odfehost in groups['odfeescontainers'] %}
      	server {{odfehost}} {{odfehost}}:9200 ssl check verify none
 {% endfor %}
@@ -53,6 +55,7 @@ listen keycloakserv
         fullconn 5000
         balance source
         option tcpka
+        option httplog
 {% for keycloakhost in groups['keycloakcontainers'] %}
 	server {{keycloakhost}} {{keycloakhost}}:8443 ssl check verify none
 {% endfor %}
@@ -65,6 +68,7 @@ listen thehiveserv
         balance source
         option tcpka
         option forwardfor
+        option httplog
 {% for thehivehost in groups['thehive'] %}
 	server {{thehivehost}} {{thehivehost}}:9000 check verify none
 {% endfor %}
@@ -77,6 +81,7 @@ listen cortexserv
         balance source
         option tcpka
         option forwardfor
+        option httplog
 {% for cortexhost in groups['cortex'] %}
 	server {{cortexhost}} {{cortexhost}}:9001 check verify none
 {% endfor %}
@@ -90,6 +95,7 @@ listen nifiservtcp77{{port}}
         balance source
         option tcpka
 	option tcp-check
+        option tcplog
 	tcp-check connect port 77{{port}}
 {% for nifihost in groups['nificontainers'] %}
    	server {{nifihost}} {{nifihost}}:77{{port}} check
@@ -106,6 +112,7 @@ listen nifiservtcp7771
         balance source
         option tcpka
 	option tcp-check
+        option tcplog
 	tcp-check connect port 7771
 {% for nifihost in groups['nificontainers'] %}
    	server {{nifihost}} {{nifihost}}:7771 check
@@ -120,6 +127,7 @@ listen nifiservhttp{{port}}
         fullconn 5000
         balance source
         option tcpka
+        option httplog
 {% for nifihost in groups['nificontainers'] %}
         server {{nifihost}} {{nifihost}}:{{port}} check
 {% endfor %}
@@ -136,6 +144,7 @@ listen nifiservtcp{{port}}
         balance source
         option tcpka
         option tcp-check
+        option tcplog
         tcp-check connect port {{port}}
 {% for nifihost in groups['nificontainers'] %}
         server {{nifihost}} {{nifihost}}:{{port}} check

roles/nifi/files/haproxy.groklib

+## These patterns were tested w/ haproxy-1.4.15
+
+## Documentation of the haproxy log formats can be found at the following links:
+## http://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat
+## http://code.google.com/p/haproxy-docs/wiki/TCPLogFormat
+
+HAPROXYTIME (?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9])
+HAPROXYDATE %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds}
+
+# Override these default patterns to parse out what is captured in your haproxy.cfg
+HAPROXYCAPTUREDREQUESTHEADERS %{DATA:captured_request_headers}
+HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers}
+
+# Example:
+#  These haproxy config lines will add data to the logs that are captured
+#  by the patterns below. Place them in your custom patterns directory to
+#  override the defaults.
+#
+#  capture request header Host len 40
+#  capture request header X-Forwarded-For len 50
+#  capture request header Accept-Language len 50
+#  capture request header Referer len 200
+#  capture request header User-Agent len 200
+#
+#  capture response header Content-Type len 30
+#  capture response header Content-Encoding len 10
+#  capture response header Cache-Control len 200
+#  capture response header Last-Modified len 200
+#
+# HAPROXYCAPTUREDREQUESTHEADERS %{DATA:request_header_host}\|%{DATA:request_header_x_forwarded_for}\|%{DATA:request_header_accept_language}\|%{DATA:request_header_referer}\|%{DATA:request_header_user_agent}
+# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified}
+
+# parse a haproxy 'httplog' line
+HAPROXYHTTPBASE %{IP:client.ip}:%{INT:client.port} \[%{HAPROXYDATE:timestamp}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server.domain} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
+
+HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
+
+# parse a haproxy 'tcplog' line
+HAPROXYTCP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client.ip}:%{INT:client.port} \[%{HAPROXYDATE:timestamp}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue}
\ No newline at end of file

roles/nifi/tasks/main.yml

@@ -129,6 +129,13 @@
   tags:
     - start
 
+- name: Copy grok libraries
+  copy:
+   src: haproxy.groklib
+   dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib"
+  tags:
+   - start
+
 - name: Create empty list of Tor nodes
   remote_user: nifi
   copy:

roles/nifi/templates/nifi.properties.j2

@@ -120,8 +120,8 @@ nifi.provenance.repository.buffer.size=100000
 
 # Component Status Repository
 nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
-nifi.components.status.repository.buffer.size=1440
-nifi.components.status.snapshot.frequency=1 min
+nifi.components.status.repository.buffer.size=288
+nifi.components.status.snapshot.frequency=5 min
 
 # Site to Site properties
 nifi.remote.input.host={{ inventory_hostname }}