From 81c0368d2ec30c673d295af794c35ea94dfae6c9 Mon Sep 17 00:00:00 2001
From: Arne Oslebo <arne.oslebo@uninett.no>
Date: Mon, 9 Nov 2020 12:26:20 +0100
Subject: [PATCH] added support for log_type label in filebeat for easier
 routing in NiFi

---
 roles/filebeat/templates/filebeat.yml.j2 |  33 +++++++
 roles/nifi/templates/flow.xml.j2         | 114 ++++++++++++++++++++---
 soctools-inventory                       |   6 +-
 3 files changed, 136 insertions(+), 17 deletions(-)
 create mode 100644 roles/filebeat/templates/filebeat.yml.j2

diff --git a/roles/filebeat/templates/filebeat.yml.j2 b/roles/filebeat/templates/filebeat.yml.j2
new file mode 100644
index 0000000..e86853f
--- /dev/null
+++ b/roles/filebeat/templates/filebeat.yml.j2
@@ -0,0 +1,33 @@
+filebeat.inputs:
+- type: log
+  paths:
+{% for file in FILEBEAT_FILES %}
+      - {{ file }}
+{% endfor %}
+  json.keys_under_root: true
+  json.overwrite_keys: true
+  json.add_error_key: true
+  fields_under_root: true
+
+setup.template.enabled: false
+setup.ilm.enabled: false
+
+output.logstash:
+  enabled: true
+  protocol: https
+  hosts: ["{{ FILEBEAT_OUTPUT_HOST }}:{{ FILEBEAT_OUTPUT_PORT }}"]
+  worker: 4
+  compression_level: 0
+  loadbalance: true
+  ssl:
+    enabled: false
+    certificate: "{{ FILEBEAT_CERT }}"
+    key: "{{ FILEBEAT_KEY }}"
+
+logging.level: info
+processors:
+ - drop_fields:
+     fields: ["beat.hostname","beat.name","beat.version","offset","pid","v"]
+ - add_fields:
+     fields:
+       log_type: {{ FILEBEAT_LOG_TYPE }}
diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2
index fe34d74..f8605d1 100644
--- a/roles/nifi/templates/flow.xml.j2
+++ b/roles/nifi/templates/flow.xml.j2
@@ -4684,11 +4684,11 @@
             </property>
             <property>
               <name>suricata</name>
-              <value>${data_type:equals("suricata")}</value>
+              <value>${log_type:equals("suricata")}</value>
             </property>
             <property>
               <name>zeek</name>
-              <value>${data_type:equals("zeek")}</value>
+              <value>${log_type:equals("zeek")}</value>
             </property>
           </processor>
           <outputPort>
@@ -4855,6 +4855,72 @@
                 <value>All</value>
               </property>
             </processor>
+            <processor>
+              <id>d3e43667-10ef-1528-b935-47c2f077f2c9</id>
+              <name>ListenBeats</name>
+              <position x="-616.0" y="-56.0" />
+              <styles />
+              <comment />
+              <class>org.apache.nifi.processors.beats.ListenBeats</class>
+              <bundle>
+                <group>org.apache.nifi</group>
+                <artifact>nifi-beats-nar</artifact>
+                <version>1.11.4</version>
+              </bundle>
+              <maxConcurrentTasks>1</maxConcurrentTasks>
+              <schedulingPeriod>0 sec</schedulingPeriod>
+              <penalizationPeriod>30 sec</penalizationPeriod>
+              <yieldPeriod>1 sec</yieldPeriod>
+              <bulletinLevel>WARN</bulletinLevel>
+              <lossTolerant>false</lossTolerant>
+              <scheduledState>RUNNING</scheduledState>
+              <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+              <executionNode>ALL</executionNode>
+              <runDurationNanos>0</runDurationNanos>
+              <property>
+                <name>Local Network Interface</name>
+              </property>
+              <property>
+                <name>Port</name>
+                <value>6001</value>
+              </property>
+              <property>
+                <name>Receive Buffer Size</name>
+                <value>65507 B</value>
+              </property>
+              <property>
+                <name>Max Size of Message Queue</name>
+                <value>10000</value>
+              </property>
+              <property>
+                <name>Max Size of Socket Buffer</name>
+                <value>2 MB</value>
+              </property>
+              <property>
+                <name>Character Set</name>
+                <value>UTF-8</value>
+              </property>
+              <property>
+                <name>Max Batch Size</name>
+                <value>10000</value>
+              </property>
+              <property>
+                <name>Message Delimiter</name>
+                <value>,\n</value>
+              </property>
+              <property>
+                <name>Max Number of TCP Connections</name>
+                <value>100</value>
+              </property>
+              <property>
+                <name>SSL_CONTEXT_SERVICE</name>
+                <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
+              </property>
+              <property>
+                <name>Client Auth</name>
+                <value>NONE</value>
+              </property>
+            </processor>
             <processor>
               <id>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</id>
               <name>Append ]</name>
@@ -5002,8 +5068,8 @@
                 <value>17b30955-5464-3709-8a32-69a459850cfa</value>
               </property>
               <property>
-                <name>data_type</name>
-                <value>/data_type</value>
+                <name>log_type</name>
+                <value>/fields/log_type</value>
               </property>
               <autoTerminatedRelationship>original</autoTerminatedRelationship>
             </processor>
@@ -5142,6 +5208,26 @@
               <partitioningAttribute />
               <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
             </connection>
+            <connection>
+              <id>ac7a988a-0175-1000-ffff-ffff86c66751</id>
+              <name />
+              <bendPoints />
+              <labelIndex>1</labelIndex>
+              <zIndex>0</zIndex>
+              <sourceId>d3e43667-10ef-1528-b935-47c2f077f2c9</sourceId>
+              <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+              <sourceType>PROCESSOR</sourceType>
+              <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId>
+              <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
+              <destinationType>PROCESSOR</destinationType>
+              <relationship>success</relationship>
+              <maxWorkQueueSize>10000</maxWorkQueueSize>
+              <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+              <flowFileExpiration>0 sec</flowFileExpiration>
+              <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+              <partitioningAttribute />
+              <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+            </connection>
             <connection>
               <id>8961779d-0175-1000-0000-00003ef237de</id>
               <name />
@@ -5430,14 +5516,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
-              <position x="283.72871497338747" y="273.4623850295515" />
-            </funnel>
             <funnel>
               <id>d8f19295-5666-31a8-b701-52214c4db51d</id>
               <position x="-1500.995244929405" y="257.20806784146276" />
             </funnel>
+            <funnel>
+              <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
+              <position x="283.72871497338747" y="273.4623850295515" />
+            </funnel>
             <processGroup>
               <id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id>
               <name>TLS events</name>
@@ -6634,14 +6720,14 @@
               <comments />
               <scheduledState>RUNNING</scheduledState>
             </outputPort>
-            <funnel>
-              <id>06521038-335b-3139-839d-ab43a013ce03</id>
-              <position x="-1557.869726298236" y="758.8984861527665" />
-            </funnel>
             <funnel>
               <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
               <position x="248.5321508445502" y="703.4412774751572" />
             </funnel>
+            <funnel>
+              <id>06521038-335b-3139-839d-ab43a013ce03</id>
+              <position x="-1557.869726298236" y="758.8984861527665" />
+            </funnel>
             <connection>
               <id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
               <name />
@@ -7378,7 +7464,7 @@
             </property>
             <property>
               <name>Password</name>
-              <value>enc{122e3a53e4a8959bb7f1b39e916fea2572e0d29014996e4288541e0fa8da0a4d65dad32242fc1817d2f07c967759113a}</value>
+              <value>enc{7b1cf5755da3904a9ab1c4b554431a538ca1409f1f9b94c41e7bab88e58aeb5528fd5bb034229065911e35d565c83d5b}</value>
             </property>
             <property>
               <name>elasticsearch-http-connect-timeout</name>
@@ -10218,7 +10304,7 @@
       </property>
       <property>
         <name>Truststore Password</name>
-        <value>enc{403bd69a9cb1b362f014b238fd0ad8dbf490d31810e3e5acb58ba40b4ba4b790}</value>
+        <value>enc{5d07dd81d60b85c07f07a91beba50b82f18a29f5846638ded1a5ce72b324fdd7}</value>
       </property>
       <property>
         <name>Truststore Type</name>
diff --git a/soctools-inventory b/soctools-inventory
index 8e0ef40..1ad3a1c 100644
--- a/soctools-inventory
+++ b/soctools-inventory
@@ -26,9 +26,9 @@ dsoclab-misp ansible_connection=docker
 dsoclab-haproxy ansible_connection=docker
 
 [filebeat]
-dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]'
-dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]'
-dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]'
+dsoclab-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi"
+dsoclab-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi"
+dsoclab-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log"]' FILEBEAT_LOG_TYPE="nifi"
 #dsoclab-odfe-1 ansible_connection=docker FILEBEAT_FILES='[""]'
 #dsoclab-odfe-2 ansible_connection=docker FILEBEAT_FILES='[""]'
 #dsoclab-kibana ansible_connection=docker FILEBEAT_FILES='[""]'
-- 
GitLab