diff --git a/inventories/filebeat b/inventories/filebeat index 9f4153ac2aa99966fc0a950156f51fe596d69065..e8df75e149c553272de4cf36dfcb71742f94c2e5 100644 --- a/inventories/filebeat +++ b/inventories/filebeat @@ -13,3 +13,4 @@ soctools-zookeeper ansible_connection=docker FILEBEAT_FILES='["/var/log/supervis soctools-cortex ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cortex" FILEBEAT_LOG_FORMAT="text" soctools-thehive ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="thehive" FILEBEAT_LOG_FORMAT="text" soctools-cassandra ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="cassandra" FILEBEAT_LOG_FORMAT="text" + diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2 index d3c0eef549b89e6b5ade7b5bef0b5bc71f884594..f35e48e961672e296b38bc91d133ede5f5fb9fd4 100644 --- a/roles/haproxy/templates/haproxy.cfg.j2 +++ b/roles/haproxy/templates/haproxy.cfg.j2 @@ -31,6 +31,7 @@ listen nifiserv fullconn 5000 balance source option tcpka + option httplog {% for nifihost in groups['nificontainers'] %} server {{nifihost}} {{nifihost}}:9443 ssl check verify none {% endfor %} @@ -42,6 +43,7 @@ listen odfeserv fullconn 5000 balance source option tcpka + option httplog {% for odfehost in groups['odfeescontainers'] %} server {{odfehost}} {{odfehost}}:9200 ssl check verify none {% endfor %} @@ -53,6 +55,7 @@ listen keycloakserv fullconn 5000 balance source option tcpka + option httplog {% for keycloakhost in groups['keycloakcontainers'] %} server {{keycloakhost}} {{keycloakhost}}:8443 ssl check verify none {% endfor %} @@ -65,6 +68,7 @@ listen thehiveserv balance source option tcpka option forwardfor + option httplog {% for thehivehost in groups['thehive'] %} server {{thehivehost}} {{thehivehost}}:9000 check verify none {% endfor %} @@ -77,6 +81,7 @@ listen cortexserv balance source option tcpka option forwardfor + option httplog {% for cortexhost in groups['cortex'] %} server {{cortexhost}} {{cortexhost}}:9001 check verify none {% endfor %} @@ -90,6 +95,7 @@ listen nifiservtcp77{{port}} balance source option tcpka option tcp-check + option tcplog tcp-check connect port 77{{port}} {% for nifihost in groups['nificontainers'] %} server {{nifihost}} {{nifihost}}:77{{port}} check @@ -106,6 +112,7 @@ listen nifiservtcp7771 balance source option tcpka option tcp-check + option tcplog tcp-check connect port 7771 {% for nifihost in groups['nificontainers'] %} server {{nifihost}} {{nifihost}}:7771 check @@ -120,6 +127,7 @@ listen nifiservhttp{{port}} fullconn 5000 balance source option tcpka + option httplog {% for nifihost in groups['nificontainers'] %} server {{nifihost}} {{nifihost}}:{{port}} check {% endfor %} @@ -136,6 +144,7 @@ listen nifiservtcp{{port}} balance source option tcpka option tcp-check + option tcplog tcp-check connect port {{port}} {% for nifihost in groups['nificontainers'] %} server {{nifihost}} {{nifihost}}:{{port}} check diff --git a/roles/nifi/files/haproxy.groklib b/roles/nifi/files/haproxy.groklib new file mode 100644 index 0000000000000000000000000000000000000000..37517af78809e9cf66aa4f1e95817457aec306ee --- /dev/null +++ b/roles/nifi/files/haproxy.groklib @@ -0,0 +1,39 @@ +## These patterns were tested w/ haproxy-1.4.15 + +## Documentation of the haproxy log formats can be found at the following links: +## http://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat +## http://code.google.com/p/haproxy-docs/wiki/TCPLogFormat + +HAPROXYTIME (?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9]) +HAPROXYDATE %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds} + +# Override these default patterns to parse out what is captured in your haproxy.cfg +HAPROXYCAPTUREDREQUESTHEADERS %{DATA:captured_request_headers} +HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers} + +# Example: +# These haproxy config lines will add data to the logs that are captured +# by the patterns below. Place them in your custom patterns directory to +# override the defaults. +# +# capture request header Host len 40 +# capture request header X-Forwarded-For len 50 +# capture request header Accept-Language len 50 +# capture request header Referer len 200 +# capture request header User-Agent len 200 +# +# capture response header Content-Type len 30 +# capture response header Content-Encoding len 10 +# capture response header Cache-Control len 200 +# capture response header Last-Modified len 200 +# +# HAPROXYCAPTUREDREQUESTHEADERS %{DATA:request_header_host}\|%{DATA:request_header_x_forwarded_for}\|%{DATA:request_header_accept_language}\|%{DATA:request_header_referer}\|%{DATA:request_header_user_agent} +# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified} + +# parse a haproxy 'httplog' line +HAPROXYHTTPBASE %{IP:client.ip}:%{INT:client.port} \[%{HAPROXYDATE:timestamp}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server.domain} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?" + +HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE} + +# parse a haproxy 'tcplog' line +HAPROXYTCP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client.ip}:%{INT:client.port} \[%{HAPROXYDATE:timestamp}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} \ No newline at end of file diff --git a/roles/nifi/tasks/main.yml b/roles/nifi/tasks/main.yml index 392a6dd38028abd302277e5093e3bdbb8a6d4567..30cb34c31c3b682f41c21336e1feae40b44a0ab9 100644 --- a/roles/nifi/tasks/main.yml +++ b/roles/nifi/tasks/main.yml @@ -129,6 +129,13 @@ tags: - start +- name: Copy grok libraries + copy: + src: haproxy.groklib + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib" + tags: + - start + - name: Create empty list of Tor nodes remote_user: nifi copy: diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2 index 8fbd37ffb0939b420f881637ec2d3bbfeb8ddd0c..8e8cc20319c706a462f7b2b4719e4e977c2b2cac 100644 --- a/roles/nifi/templates/flow.xml.j2 +++ b/roles/nifi/templates/flow.xml.j2 @@ -2759,8 +2759,8 @@ <value>/opt/nifi/nifi-current/conf/enrich/freq</value> </property> <property> - <name>Frequency File</name> - <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value> + <name>record-reader</name> + <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> </property> <property> <name>record-writer</name> @@ -2771,16 +2771,16 @@ <value>${enrich_domain1}</value> </property> <property> - <name>record-reader</name> - <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> + <name>Result Field 1</name> + <value>${enrich_domain1}_freq1</value> </property> <property> <name>Result Field 2</name> <value>${enrich_domain1}_freq2</value> </property> <property> - <name>Result Field 1</name> - <value>${enrich_domain1}_freq1</value> + <name>Frequency File</name> + <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value> </property> </processor> <inputPort> @@ -3432,8 +3432,8 @@ <value>/opt/nifi/nifi-current/conf/enrich/freq</value> </property> <property> - <name>Frequency File</name> - <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value> + <name>record-reader</name> + <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> </property> <property> <name>record-writer</name> @@ -3444,16 +3444,16 @@ <value>${enrich_domain2}</value> </property> <property> - <name>record-reader</name> - <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> + <name>Result Field 1</name> + <value>${enrich_domain2}_freq1</value> </property> <property> <name>Result Field 2</name> <value>${enrich_domain2}_freq2</value> </property> <property> - <name>Result Field 1</name> - <value>${enrich_domain2}_freq1</value> + <name>Frequency File</name> + <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value> </property> </processor> <inputPort> @@ -4205,16 +4205,16 @@ <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <outputPort> - <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id> - <name>To enrichment</name> - <position x="480.0" y="392.0" /> + <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id> + <name>To data output</name> + <position x="-1120.0" y="592.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> <outputPort> - <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id> - <name>To data output</name> - <position x="-1120.0" y="592.0" /> + <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id> + <name>To enrichment</name> + <position x="480.0" y="392.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> @@ -4226,16 +4226,16 @@ <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <outputPort> - <id>27d5761b-0172-1000-0000-000059275dad</id> - <name>To enrichment</name> - <position x="-312.0" y="328.0" /> + <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id> + <name>To data output</name> + <position x="-632.0" y="328.0" /> <comments /> <scheduledState>STOPPED</scheduledState> </outputPort> <outputPort> - <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id> - <name>To data output</name> - <position x="-632.0" y="328.0" /> + <id>27d5761b-0172-1000-0000-000059275dad</id> + <name>To enrichment</name> + <position x="-312.0" y="328.0" /> <comments /> <scheduledState>STOPPED</scheduledState> </outputPort> @@ -4277,6 +4277,10 @@ <name>suricata</name> <value>${log_type:equals("suricata")}</value> </property> + <property> + <name>haproxy</name> + <value>${log_type:equals("haproxy")}</value> + </property> <property> <name>zeek</name> <value>${log_type:equals("zeek")}</value> @@ -4287,16 +4291,16 @@ </property> </processor> <outputPort> - <id>bcb879d5-0175-1000-0000-000070879ad0</id> - <name>To data output</name> - <position x="-2480.0" y="336.0" /> + <id>349b32fe-a821-1197-0000-00003a0b6fe5</id> + <name>To enrichment</name> + <position x="360.0" y="424.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> <outputPort> - <id>349b32fe-a821-1197-0000-00003a0b6fe5</id> - <name>To enrichment</name> - <position x="352.0" y="296.0" /> + <id>bcb879d5-0175-1000-0000-000070879ad0</id> + <name>To data output</name> + <position x="-2480.0" y="336.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> @@ -4582,14 +4586,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>bc925474-0175-1000-0000-00004e78071f</id> - <position x="1882.9999517774115" y="327.9999931568573" /> - </funnel> <funnel> <id>bc90d189-0175-1000-0000-0000037bc986</id> <position x="8.0" y="424.0" /> </funnel> + <funnel> + <id>bc925474-0175-1000-0000-00004e78071f</id> + <position x="1882.9999517774115" y="327.9999931568573" /> + </funnel> <connection> <id>bc90c7ac-0175-1000-ffff-fffffa80b534</id> <name /> @@ -5096,14 +5100,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>895f7db3-0175-1000-ffff-ffff8229d688</id> - <position x="-1446.1517058240609" y="301.4492766741185" /> - </funnel> <funnel> <id>895faa7a-0175-1000-0000-000014ef9dd3</id> <position x="278.84829417593915" y="332.4492766741185" /> </funnel> + <funnel> + <id>895f7db3-0175-1000-ffff-ffff8229d688</id> + <position x="-1446.1517058240609" y="301.4492766741185" /> + </funnel> <connection> <id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id> <name /> @@ -6568,6 +6572,350 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> </processGroup> + <processGroup> + <id>5d04357e-423c-1ab5-a7a4-44565abfed7f</id> + <name>Haproxy</name> + <position x="-448.0" y="664.0" /> + <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> + <processor> + <id>c9763c4c-7186-1460-871a-b5fd00ca3241</id> + <name>UpdateRecord</name> + <position x="352.0" y="472.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.UpdateRecord</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>record-reader</name> + <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> + </property> + <property> + <name>record-writer</name> + <value>17b30955-5464-3709-8a32-69a459850cfa</value> + </property> + <property> + <name>replacement-value-strategy</name> + <value>literal-value</value> + </property> + <property> + <name>/labels/source_host</name> + <value>${source_host}</value> + </property> + <property> + <name>/timestamp</name> + <value>${field.value:toDate('dd/MMM/yyyy:HH:mm:ss.SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value> + </property> + </processor> + <processor> + <id>e4c8356d-54ad-15b5-94fe-799d9465aa51</id> + <name>Extract message</name> + <position x="352.0" y="280.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.ConvertRecord</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>record-reader</name> + <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> + </property> + <property> + <name>record-writer</name> + <value>bc8e5957-0175-1000-0000-00003346421d</value> + </property> + <property> + <name>include-zero-record-flowfiles</name> + <value>true</value> + </property> + </processor> + <processor> + <id>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</id> + <name>Convert to json</name> + <position x="1064.0" y="272.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.ConvertRecord</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>record-reader</name> + <value>56ebe0aa-0176-1000-ffff-ffffbd212f01</value> + </property> + <property> + <name>record-writer</name> + <value>17b30955-5464-3709-8a32-69a459850cfa</value> + </property> + <property> + <name>include-zero-record-flowfiles</name> + <value>false</value> + </property> + </processor> + <processor> + <id>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</id> + <name>UpdateAttribute</name> + <position x="1072.0" y="472.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.attributes.UpdateAttribute</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-update-attribute-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Delete Attributes Expression</name> + </property> + <property> + <name>Store State</name> + <value>Do not store state</value> + </property> + <property> + <name>Stateful Variables Initial Value</name> + </property> + <property> + <name>canonical-value-lookup-cache-size</name> + <value>100</value> + </property> + <property> + <name>data_index</name> + <value>logs-haproxy</value> + </property> + </processor> + <inputPort> + <id>65a33e05-e157-1bfc-8741-adf11b3df720</id> + <name>Input</name> + <position x="397.9999517774115" y="110.99999315685733" /> + <comments /> + <scheduledState>RUNNING</scheduledState> + </inputPort> + <outputPort> + <id>328b35e2-eb52-1f47-b84d-52941eff8a07</id> + <name>Output</name> + <position x="1120.0" y="808.0" /> + <comments /> + <scheduledState>RUNNING</scheduledState> + </outputPort> + <funnel> + <id>bb763b6c-302d-12a4-8eb2-b3b501d92244</id> + <position x="1882.9999517774115" y="327.9999931568573" /> + </funnel> + <funnel> + <id>312d3490-461e-13ac-a3a2-603704c456e2</id> + <position x="8.0" y="424.0" /> + </funnel> + <connection> + <id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>328b35e2-eb52-1f47-b84d-52941eff8a07</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>OUTPUT_PORT</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>0ecb3e12-768e-1896-a850-2a2bec52eb95</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>9451307c-96df-1302-8189-8e556060bb80</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bb763b6c-302d-12a4-8eb2-b3b501d92244</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>fbd23518-c750-15f2-b8e0-9a0600eb3246</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>e4c8356d-54ad-15b5-94fe-799d9465aa51</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>312d3490-461e-13ac-a3a2-603704c456e2</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>60 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>b5d43cea-5555-10b0-b75f-b88a95e9c6aa</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>65a33e05-e157-1bfc-8741-adf11b3df720</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>INPUT_PORT</sourceType> + <destinationId>e4c8356d-54ad-15b5-94fe-799d9465aa51</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship /> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>1285380e-3d8e-199c-a6b2-fec085b63cd4</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>312d3490-461e-13ac-a3a2-603704c456e2</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>60 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>484a3eab-4af3-11cd-abe2-d5ee6fc1a291</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>e4c8356d-54ad-15b5-94fe-799d9465aa51</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>39ef3a2d-874e-11a6-87be-0b3582fa43de</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>c9763c4c-7186-1460-871a-b5fd00ca3241</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + </processGroup> <processGroup> <id>83691174-683f-3c7c-8526-8fc00397aee1</id> <name>Zeek</name> @@ -6784,14 +7132,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>06521038-335b-3139-839d-ab43a013ce03</id> - <position x="-1557.869726298236" y="758.8984861527665" /> - </funnel> <funnel> <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id> <position x="248.5321508445502" y="703.4412774751572" /> </funnel> + <funnel> + <id>06521038-335b-3139-839d-ab43a013ce03</id> + <position x="-1557.869726298236" y="758.8984861527665" /> + </funnel> <connection> <id>3c739604-b69c-3e86-ba4c-a4739078837c</id> <name /> @@ -6973,6 +7321,48 @@ <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> + <connection> + <id>56e5add3-0176-1000-ffff-ffffd667d1f6</id> + <name /> + <bendPoints> + <bendPoint x="-584.0" y="624.0" /> + </bendPoints> + <labelIndex>0</labelIndex> + <zIndex>0</zIndex> + <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId> + <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>65a33e05-e157-1bfc-8741-adf11b3df720</destinationId> + <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId> + <destinationType>INPUT_PORT</destinationType> + <relationship>haproxy</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>56e5f029-0176-1000-ffff-fffff7512a3b</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId> + <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId> + <sourceType>OUTPUT_PORT</sourceType> + <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId> + <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId> + <destinationType>OUTPUT_PORT</destinationType> + <relationship /> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> <connection> <id>89630460-0175-1000-0000-00006b5f18c8</id> <name /> @@ -7606,7 +7996,7 @@ </property> <property> <name>Password</name> - <value>enc{a2a2e69fa7a4563faa60a9e069ae100df4117e257dfa1cdbe21ed388bc4480066d147f741a89f88b121b598a508212f6}</value> + <value>enc{1c9a67efa861b9a5f0ced47e1bb930650b19b788b8576e55d87fa2a3a4760d790d7425f299ed70ea1859a64a26753959}</value> </property> <property> <name>elasticsearch-http-connect-timeout</name> @@ -10618,7 +11008,7 @@ </property> <property> <name>Truststore Password</name> - <value>enc{03ca02479138f0ae3484435afa13a8d10bdc66b2b9391e10764a3f022ad1f693}</value> + <value>enc{d064a1e3a5a974d37b0202bbb9551137b9543af176d965ad630f0fc2bdafa690}</value> </property> <property> <name>Truststore Type</name> @@ -10629,6 +11019,51 @@ <value>TLS</value> </property> </controllerService> + <controllerService> + <id>56ebe0aa-0176-1000-ffff-ffffbd212f01</id> + <name>Haproxy GrokReader</name> + <comment /> + <class>org.apache.nifi.grok.GrokReader</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-record-serialization-services-nar</artifact> + <version>1.12.1</version> + </bundle> + <enabled>true</enabled> + <property> + <name>schema-access-strategy</name> + <value>string-fields-from-grok-expression</value> + </property> + <property> + <name>schema-registry</name> + </property> + <property> + <name>schema-name</name> + <value>${schema.name}</value> + </property> + <property> + <name>schema-version</name> + </property> + <property> + <name>schema-branch</name> + </property> + <property> + <name>schema-text</name> + <value>${avro.schema}</value> + </property> + <property> + <name>Grok Pattern File</name> + <value>/opt/nifi/nifi-current/conf/enrich/haproxy.groklib</value> + </property> + <property> + <name>Grok Expression</name> + <value>%{PROG:process.name}(?:\[%{POSINT:process.pid}\])?: %{HAPROXYHTTPBASE}</value> + </property> + <property> + <name>no-match-behavior</name> + <value>append-to-previous-message</value> + </property> + </controllerService> <controllerService> <id>179dd31f-89ed-3179-adb2-85a9c61869ce</id> <name>Inferred JsonTreeReader</name> diff --git a/roles/nifi/templates/nifi.properties.j2 b/roles/nifi/templates/nifi.properties.j2 index 6efbc1e3169fb8f8e1f7422b5d8cb4627697782e..188f234fb5ac788f5e89c0f5fc42251fe100b771 100644 --- a/roles/nifi/templates/nifi.properties.j2 +++ b/roles/nifi/templates/nifi.properties.j2 @@ -120,8 +120,8 @@ nifi.provenance.repository.buffer.size=100000 # Component Status Repository nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository -nifi.components.status.repository.buffer.size=1440 -nifi.components.status.snapshot.frequency=1 min +nifi.components.status.repository.buffer.size=288 +nifi.components.status.snapshot.frequency=5 min # Site to Site properties nifi.remote.input.host={{ inventory_hostname }}