From 81c0368d2ec30c673d295af794c35ea94dfae6c9 Mon Sep 17 00:00:00 2001 From: Arne Oslebo <arne.oslebo@uninett.no> Date: Mon, 9 Nov 2020 12:26:20 +0100 Subject: [PATCH] added support for log_type label in filebeat for easier routing in NiFi --- roles/filebeat/templates/filebeat.yml.j2 | 33 +++++++ roles/nifi/templates/flow.xml.j2 | 114 ++++++++++++++++++++--- soctools-inventory | 6 +- 3 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 roles/filebeat/templates/filebeat.yml.j2 diff --git a/roles/filebeat/templates/filebeat.yml.j2 b/roles/filebeat/templates/filebeat.yml.j2 new file mode 100644 index 0000000..e86853f --- /dev/null +++ b/roles/filebeat/templates/filebeat.yml.j2 @@ -0,0 +1,33 @@ +filebeat.inputs: +- type: log + paths: +{% for file in FILEBEAT_FILES %} + - {{ file }} +{% endfor %} + json.keys_under_root: true + json.overwrite_keys: true + json.add_error_key: true + fields_under_root: true + +setup.template.enabled: false +setup.ilm.enabled: false + +output.logstash: + enabled: true + protocol: https + hosts: ["{{ FILEBEAT_OUTPUT_HOST }}:{{ FILEBEAT_OUTPUT_PORT }}"] + worker: 4 + compression_level: 0 + loadbalance: true + ssl: + enabled: false + certificate: "{{ FILEBEAT_CERT }}" + key: "{{ FILEBEAT_KEY }}" + +logging.level: info +processors: + - drop_fields: + fields: ["beat.hostname","beat.name","beat.version","offset","pid","v"] + - add_fields: + fields: + log_type: {{ FILEBEAT_LOG_TYPE }} diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2 index fe34d74..f8605d1 100644 --- a/roles/nifi/templates/flow.xml.j2 +++ b/roles/nifi/templates/flow.xml.j2 @@ -4684,11 +4684,11 @@ </property> <property> <name>suricata</name> - <value>${data_type:equals("suricata")}</value> + <value>${log_type:equals("suricata")}</value> </property> <property> <name>zeek</name> - <value>${data_type:equals("zeek")}</value> + <value>${log_type:equals("zeek")}</value> </property> </processor> <outputPort> @@ -4855,6 +4855,72 @@ <value>All</value> </property> </processor> + <processor> + <id>d3e43667-10ef-1528-b935-47c2f077f2c9</id> + <name>ListenBeats</name> + <position x="-616.0" y="-56.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.beats.ListenBeats</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-beats-nar</artifact> + <version>1.11.4</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Local Network Interface</name> + </property> + <property> + <name>Port</name> + <value>6001</value> + </property> + <property> + <name>Receive Buffer Size</name> + <value>65507 B</value> + </property> + <property> + <name>Max Size of Message Queue</name> + <value>10000</value> + </property> + <property> + <name>Max Size of Socket Buffer</name> + <value>2 MB</value> + </property> + <property> + <name>Character Set</name> + <value>UTF-8</value> + </property> + <property> + <name>Max Batch Size</name> + <value>10000</value> + </property> + <property> + <name>Message Delimiter</name> + <value>,\n</value> + </property> + <property> + <name>Max Number of TCP Connections</name> + <value>100</value> + </property> + <property> + <name>SSL_CONTEXT_SERVICE</name> + <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value> + </property> + <property> + <name>Client Auth</name> + <value>NONE</value> + </property> + </processor> <processor> <id>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</id> <name>Append ]</name> @@ -5002,8 +5068,8 @@ <value>17b30955-5464-3709-8a32-69a459850cfa</value> </property> <property> - <name>data_type</name> - <value>/data_type</value> + <name>log_type</name> + <value>/fields/log_type</value> </property> <autoTerminatedRelationship>original</autoTerminatedRelationship> </processor> @@ -5142,6 +5208,26 @@ <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> + <connection> + <id>ac7a988a-0175-1000-ffff-ffff86c66751</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>d3e43667-10ef-1528-b935-47c2f077f2c9</sourceId> + <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId> + <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> <connection> <id>8961779d-0175-1000-0000-00003ef237de</id> <name /> @@ -5430,14 +5516,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id> - <position x="283.72871497338747" y="273.4623850295515" /> - </funnel> <funnel> <id>d8f19295-5666-31a8-b701-52214c4db51d</id> <position x="-1500.995244929405" y="257.20806784146276" /> </funnel> + <funnel> + <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id> + <position x="283.72871497338747" y="273.4623850295515" /> + </funnel> <processGroup> <id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id> <name>TLS events</name> @@ -6634,14 +6720,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>06521038-335b-3139-839d-ab43a013ce03</id> - <position x="-1557.869726298236" y="758.8984861527665" /> - </funnel> <funnel> <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id> <position x="248.5321508445502" y="703.4412774751572" /> </funnel> + <funnel> + <id>06521038-335b-3139-839d-ab43a013ce03</id> + <position x="-1557.869726298236" y="758.8984861527665" /> + </funnel> <connection> <id>3c739604-b69c-3e86-ba4c-a4739078837c</id> <name /> @@ -7378,7 +7464,7 @@ </property> <property> <name>Password</name> - <value>enc{122e3a53e4a8959bb7f1b39e916fea2572e0d29014996e4288541e0fa8da0a4d65dad32242fc1817d2f07c967759113a}</value> + <value>enc{7b1cf5755da3904a9ab1c4b554431a538ca1409f1f9b94c41e7bab88e58aeb5528fd5bb034229065911e35d565c83d5b}</value> </property> <property> <name>elasticsearch-http-connect-timeout</name> @@ -10218,7 +10304,7 @@ </property> <property> <name>Truststore Password</name> - <value>enc{403bd69a9cb1b362f014b238fd0ad8dbf490d31810e3e5acb58ba40b4ba4b790}</value> + <value>enc{5d07dd81d60b85c07f07a91beba50b82f18a29f5846638ded1a5ce72b324fdd7}</value> </property> <property> <name>Truststore Type</name> diff --git a/soctools-inventory b/soctools-inventory index 8e0ef40..1ad3a1c 100644 --- a/soctools-inventory +++ b/soctools-inventory @@ -26,9 +26,9 @@ dsoclab-misp ansible_connection=docker dsoclab-haproxy ansible_connection=docker [filebeat] -dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' -dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' -dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' +dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" +dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" +dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi" #dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='[""]' #dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='[""]' #dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='[""]' -- GitLab