upgraded NiFi and filebeat to new versions. Fixed various issues with sending...

upgraded NiFi and filebeat to new versions. Fixed various issues with sending logs with filebeat to nifi

group_vars/all/main.yml

@@ -20,28 +20,11 @@ haproxy_img: "{{repo}}/haproxy:{{version}}{{suffix}}"
 HAPROXY_PROCESSES: "2"
 HAPROXY_STATS_PASS: "eiph2Eepaizicheelah3tei+bae3ohgh"
 
-FILEBEAT_VERSION: "7.4.1"
+FILEBEAT_VERSION: "7.9.3"
 FILEBEAT_OUTPUT_HOST: "{{dslproxy}}"
-FILEBEAT_OUTPUT_PORT: "5019"
+FILEBEAT_OUTPUT_PORT: "6000"
 FILEBEAT_CERT: "/opt/filebeat/filebeat.crt"
 FILEBEAT_KEY: "/opt/filebeat/filebeat.key"
-FILEBEAT_NIFI_FILES:
-  - /var/log/*.log
-FILEBEAT_HAPROXY_FILES:
-  - /var/log/*.log
-FILEBEAT_KEYCLOAK_FILES:
-  - /var/log/*.log
-FILEBEAT_MISP_FILES:
-  - /var/log/*.log
-FILEBEAT_MYSQL_FILES:
-  - /var/log/*.log
-FILEBEAT_ODFEES_FILES:
-  - /var/log/*.log
-FILEBEAT_ODFEKIBANA_FILES:
-  - /var/log/*.log
-FILEBEAT_ZOOKEEPER_FILES:
-  - /var/log/*.log
-
 
 temp_root: "/tmp/centosbuild"
 
@@ -63,14 +46,14 @@ kspass: "Testing003"
 tspass: "Testing003"
 
 sysctlconfig:
-  - { key: "net.core.rmem_max", val: "2097152" }
-  - { key: "net.core.wmem_max", val: "2097152" }
+  - { key: "net.core.rmem_max", val: "4194304" }
+  - { key: "net.core.wmem_max", val: "4194304" }
   - { key: "vm.max_map_count" , val:  "524288" }
 
 nifi_javamem: "4g"
 odfe_javamem: "2g"
 
-nifi_version: 1.11.4
+nifi_version: 1.12.1
 nifi_repo: "https://archive.apache.org/dist"
 
 ca_cn: "SOCTOOLS-CA"

roles/filebeat/templates/filebeat.yml.j2

@@ -4,30 +4,17 @@ filebeat.inputs:
 {% for file in FILEBEAT_FILES %}
       - {{ file }}
 {% endfor %}
-  json.keys_under_root: true
-  json.overwrite_keys: true
-  json.add_error_key: true
-  fields_under_root: true
+  fields:
+    log_type: {{ FILEBEAT_LOG_TYPE }}
 
-setup.template.enabled: false
-setup.ilm.enabled: false
+filebeat.config.modules:
+  path: ${path.config}/modules.d/*.yml
+  reload.enabled: false
 
 output.logstash:
-  enabled: true
-  protocol: https
   hosts: ["{{ FILEBEAT_OUTPUT_HOST }}:{{ FILEBEAT_OUTPUT_PORT }}"]
-  worker: 4
-  compression_level: 0
+  workers: 3
   loadbalance: true
-  ssl:
-    enabled: false
-    certificate: "{{ FILEBEAT_CERT }}"
-    key: "{{ FILEBEAT_KEY }}"
-
-logging.level: info
-processors:
- - drop_fields:
-     fields: ["beat.hostname","beat.name","beat.version","offset","pid","v"]
- - add_fields:
-     fields:
-       log_type: {{ FILEBEAT_LOG_TYPE }}
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+  #ssl.certificate: "{{ FILEBEAT_CERT }}""
+  #ssl.key: "{{ FILEBEAT_KEY }}"