diff --git a/README.md b/README.md
index f44a4984173bae749d95a9cbcf4551c6461643e6..967fa3f7e48937ed76b2e4a61ff2d77fa888801c 100644
--- a/README.md
+++ b/README.md
@@ -13,45 +13,36 @@ Log in and install ansible:
`yum -y install ansible git`
`ansible-galaxy collection install ansible.posix`
-Clone soctools:
-Temporary solution: Upload your ssh key to gitlab.geant.org
-`git clone git@gitlab.geant.org:gn4-3-wp8-t3.1-soc/soctools.git`
+Clone soctools:
+`git clone https://scm.uninett.no/geant-wp8-t3.1/soctools.git`
`cd soctools`
Install soctools:
-Edit group_vars/all/main.yml and change 'soctoolsproxy' so that it point to the FQDN of the server.
+Edit group_vars/all/main.yml and change 'dslproxy' so that it point to the FQDN of the server.
`vi group_vars/all/main.yml`
-Users are specified in the file:
-`group_vars/all/users.yml`
+The first entry in the soctools_users variable is the user with full admin privileges in NiFi and Kibana.
To configure the server running soctools, run the ansible playbook:
-`ansible-playbook -i inventories soctools_server.yml`
+`ansible-playbook -i soctools-inventory soctools_server.yml`
To build the Docker images needed, run the ansible playbook:
-`ansible-playbook -i inventories buildimages.yml`
+`ansible-playbook -i soctools-inventory buildimages.yml`
To build the CA needed for host and user certificates, run the ansible playbook:
-`ansible-playbook -i inventories buildca.yml`
+`ansible-playbook -i soctools-inventory buildca.yml`
-If using soctools CA certificates provided with this installation, you first need to download and import root certificate found in secrets/CA/ca.crt
-For Windows, CA certificate should be installed in Trusted Root Certification Authorities store.
-
-User certificates are can be found in the directory secrets/certificates. Import into browser for authentication.
-For Windows, user certificate should be installed in Personal store. Passwords for the certificates can be found in the directory secrets/passwords.
+User certificates are can be found in the directory roles/ca/files/CA/private. Import into browser for authentication.
To start the cluster, run the ansible playbook soctools.yml:
-`ansible-playbook -i inventories soctools.yml -t start`
+`ansible-playbook -i soctools-inventory soctools.yml -t start`
To stop the cluster, run the ansible playbook soctools.yml:
-`ansible-playbook -i inventories soctools.yml -t stop`
-
-Web interfaces are available on the following ports:
- * 9443 - NiFi
- * 5601 - Kibana
- * 6443 - Misp : Default user/password: admin@admin.test/test
- * 9000 - The Hive : Default user/password: admin@thehive.local/secret
- * 9001 - Cortex
- * 12443 - Keycloak : Default user/password: admin/Pass005
+`ansible-playbook -i soctools-inventory soctools.yml -t stop`
+
+The NiFi interface should now be available on port 9443 on the server.
+The OpenDistro for Elasticsearch interface should now be available on port 5601 on the server. To access preconfigured
+index patterns you have to switch to Global tenant.
+The Keycloak IdP interface should now be available on port 12443 on the server.
License
-------
diff --git a/buildca.yml b/buildca.yml
index 37ef902ce7086d5c92610d72eb86fa0e5022aec6..b718286139aa329e0a343f4ff60624cdfc6b3c34 100644
--- a/buildca.yml
+++ b/buildca.yml
@@ -1,7 +1,7 @@
---
- name: Build certification authority
- hosts: soctoolsmain
+ hosts: dsldev
roles:
- ca
diff --git a/group_vars/all/main.yml b/group_vars/all/main.yml
index 0c1c3d27ad556cea5d86f677becc8dddc0658312..c6adf5f95e0594287ec6815a4f43e6a95713922e 100644
--- a/group_vars/all/main.yml
+++ b/group_vars/all/main.yml
@@ -1,75 +1,124 @@
---
-soctoolsproxy: "<CHANGE_ME:hostname>"
-
-maxmind_key: ""
-
-docker_build_dir: "{{playbook_dir}}/build"
+dslproxy: "dsoclab.gn4-3-wp8-soc.sunet.se"
# TheHive Button plugin
THEHIVE_URL: "https://hive.gn4-3-wp8-soc.sunet.se/"
-THEHIVE_API_KEY: "5LymseWiurZBrQN8Kqp8O+9KniTL5cE0"
-THEHIVE_OWNER: "admin"
+# here enter API key for default admin user
+THEHIVE_API_KEY: "bs2Jc3tGJqhVv0AYyX2NYlhMlorPz7mX"
+# ID of the default admin user
+THEHIVE_OWNER: "admin@thehive.local"
+
+# TheHive Create Organisation and Users
+# Login as default admin user and create API key, populate it here
+# thehive_admin_api: "KoHrKbIJm8XMsJxA9nZLs6YemCu76o3u"
+# thehive_writer: "[write]"
+
+#THEHIVE_API_KEY: "1gFdNhmUSxO3BRe1SBB5JYEvkW9UOo6s"
+THEHIVE_USERS:
+ - kiril:
+ username: "kiril"
+ name: "Kiril"
+ surname: "Kiroski"
+ roles: '["read", "write", "admin"]'
+ organization: "uninett.no"
+ - temur:
+ username: "temur"
+ name: "Temur"
+ surname: "Maisuradze"
+ roles: '["read", "write", "admin"]'
+ organization: "uninett.no"
+
+
soctools_netname: "soctoolsnet"
soctools_network: "172.22.0.0/16"
-repo: soctools
+repo: gn43-dsl
version: 7
suffix: a20201004
-haproxy_name: "soctools-haproxy"
+haproxy_name: "dsoclab-haproxy"
haproxy_version: "2.2"
haproxy_img: "{{repo}}/haproxy:{{version}}{{suffix}}"
HAPROXY_PROCESSES: "2"
-
-FILEBEAT_VERSION: "7.9.3"
-FILEBEAT_OUTPUT_HOST: "{{soctoolsproxy}}"
-FILEBEAT_OUTPUT_PORT: "6000"
-FILEBEAT_CERT: "/opt/filebeat/filebeat.crt"
-FILEBEAT_KEY: "/opt/filebeat/filebeat.key"
+HAPROXY_STATS_PASS: "eiph2Eepaizicheelah3tei+bae3ohgh"
temp_root: "/tmp/centosbuild"
openjdk_img: "{{repo}}/openjdk:{{version}}{{suffix}}"
-zookeeper_name: "soctools-zookeeper"
+zookeeper_name: "dsoclab-zookeeper"
zookeeper_img: "{{repo}}/zookeeper:{{version}}{{suffix}}"
-misp_name: "soctools-misp"
+misp_name: "dsoclab-misp"
misp_img: "{{repo}}/misp:{{version}}{{suffix}}"
-misp_url: "https://{{soctoolsproxy}}:6443"
nifi_img: "{{repo}}/nifi:{{version}}{{suffix}}"
-mysql_name: "soctools-mysql"
+mysql_name: "dsoclab-mysql"
mysql_img: "{{repo}}/mysql:{{version}}{{suffix}}"
+mysql_dbrootpass: "Pass006"
-cassandra_name: "soctools-cassandra"
+cassandra_name: "dsoclab-cassandra"
cassandra_img: "{{repo}}/cassandra:{{version}}{{suffix}}"
-thehive_name: "soctools-thehive"
+thehive_name: "dsoclab-thehive"
thehive_img: "{{repo}}/thehive:{{version}}{{suffix}}"
+# GENERATED WITH cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1
+thehive_secret_key: "LcnI9eKLo33711BmCnzf6UM1y05pdmj3dlADL81PxuffWqhobRoiiGFftjNPKpmM"
-cortex_name: "soctools-cortex"
+cortex_name: "dsoclab-cortex"
cortex_img: "{{repo}}/cortex:{{version}}{{suffix}}"
cortex_elasticsearch_mem: "256m"
+# GENERATED WITH cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1
+cortex_secret_key: "9CZ844IcAp5dHjsgU4iuaEssdopLcS6opzhVP3Ys4t4eRpNlHmwZdtfveLEXpM9D"
+cortex_odfe_pass: "Pass009"
+
+kspass: "Testing003"
+tspass: "Testing003"
sysctlconfig:
- - { key: "net.core.rmem_max", val: "4194304" }
- - { key: "net.core.wmem_max", val: "4194304" }
+ - { key: "net.core.rmem_max", val: "2097152" }
+ - { key: "net.core.wmem_max", val: "2097152" }
- { key: "vm.max_map_count" , val: "524288" }
nifi_javamem: "1g"
odfe_javamem: "512m"
-nifi_version: 1.12.1
+nifi_version: 1.11.4
nifi_repo: "https://archive.apache.org/dist"
ca_cn: "SOCTOOLS-CA"
+soctools_users:
+ - firstname: "Bozidar"
+ lastname: "Proevski"
+ username: "bozidar.proevski"
+ email: "bozidar.proevski@finki.ukim.mk"
+ DN: "CN=Bozidar Proevski"
+ CN: "Bozidar Proevski"
+ password: "Pass001"
+ - firstname: "Arne"
+ lastname: "Oslebo"
+ username: "arne.oslebo"
+ email: "arne.oslebo@uninett.no"
+ DN: "CN=Arne Oslebo"
+ CN: "Arne Oslebo"
+ password: "Pass002"
+ - firstname: "Kiril"
+ lastname: "Kjiroski"
+ username: "kiril.kjiroski"
+ email: "kiril.kjiroski@finki.ukim.mk"
+ DN: "CN=Kiril Kjiroski"
+ CN: "Kiril Kjiroski"
+ password: "Pass003"
+
odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}"
odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}"
+# GENERATE 32-bit secure value
+odfekibana_cookie: "iroAm0ueIV7w6CS1WcJTwIV6R4d5RIAt"
+odfees_adminpass: "Pass004"
#elk_version: "oss-7.6.1"
elk_version: "oss-7.4.2"
#odfeplugin_version: "1.7.0.0"
@@ -80,25 +129,16 @@ openid_scope: profile
openid_subjkey: preferred_username
keycloak_img: "{{repo}}/keycloak:{{version}}{{suffix}}"
+keycloak_adminpass: "Pass005"
elastic_username: "admin"
+misp_token: ""
+misp_url: ""
+maxmind_key: ""
misp_dbname: "mispdb"
misp_dbuser: "misp"
-
-services:
- - mysql
- - haproxy
- - openjdk
- - zookeeper
- - nifi
- - elasticsearch
- - kibana
- - odfees
- - odfekibana
- - keycloak
- - misp
- - cassandra
- - thehive
- - cortex
+misp_dbpass: "Pass007"
+# misp_salt generated with: openssl rand -base64 32
+misp_salt: "wa2fJA2mGIn32IDl+uKrCJ069Mg3khDdGzFNv8DOwM0="
diff --git a/roles/build/tasks/centos.yml b/roles/build/tasks/centos.yml
index 62a8fb1fbd84dce5a27660a24f9eab54b59d9b97..c7be287621104cc8eb257bf0734a5c2641b58c33 100644
--- a/roles/build/tasks/centos.yml
+++ b/roles/build/tasks/centos.yml
@@ -5,115 +5,96 @@
name: "{{repo}}/centos:{{version}}{{suffix}}"
register: centosimg
-- name: Assert CentOS image
- assert:
- that: centosimg.images | length == 0
- fail_msg: "CentOS image already exists"
-
-- name: Create etc tree in build directory
- file:
- path: '{{ temp_root}}/{{ item.path }}'
- state: directory
- mode: '{{ item.mode }}'
- with_filetree: templates/etcroot/
- when: item.state == 'directory'
-
-- name: Populate etc tree in build directory
- template:
- src: '{{ item.src }}'
- dest: '{{ temp_root}}/{{ item.path }}'
- force: yes
- with_filetree: templates/etcroot
- when: item.state == 'file'
-
-- name: Create dev tree in build directory
- command: mknod -m {{ item.mode }} {{ item.dev }} {{ item.type }} {{ item.major }} {{ item.minor }}
- args:
- creates: "{{ item.dev }}"
- with_items:
- - { mode: 600, dev: "{{temp_root}}/dev/console", type: c, major: 5, minor: 1 }
- - { mode: 600, dev: "{{temp_root}}/dev/initctl", type: p, major: '', minor: '' }
- - { mode: 666, dev: "{{temp_root}}/dev/full", type: c, major: 1, minor: 7 }
- - { mode: 666, dev: "{{temp_root}}/dev/null", type: c, major: 1, minor: 3 }
- - { mode: 666, dev: "{{temp_root}}/dev/ptmx", type: c, major: 5, minor: 2 }
- - { mode: 666, dev: "{{temp_root}}/dev/random", type: c, major: 1, minor: 8 }
- - { mode: 666, dev: "{{temp_root}}/dev/tty", type: c, major: 5, minor: 0 }
- - { mode: 666, dev: "{{temp_root}}/dev/tty0", type: c, major: 4, minor: 0 }
- - { mode: 666, dev: "{{temp_root}}/dev/urandom", type: c, major: 1, minor: 9 }
- - { mode: 666, dev: "{{temp_root}}/dev/zero", type: c, major: 1, minor: 5 }
-
-- name: Install centos-release in build directory
- yum:
- installroot: "{{ temp_root}}"
- name: centos-release
- state: present
-
-- name: Install Core CentOS in build directory
- yum:
- installroot: "{{ temp_root}}"
- name:
- - "@Core"
- - yum-plugin-ovl.noarch
- - epel-release
- state: present
-
-- name: Install extra packages
- yum:
- installroot: "{{ temp_root }}"
- name:
- - daemonize
- state: present
-
-- name: Clean yum cache
- command: 'yum --installroot="{{ temp_root}}" -y clean all'
-
-- name: Remove unneeded directories
- file:
- path: "{{temp_root}}/{{item}}"
- state: absent
- with_items:
- - usr/share/cracklib
- - var/cache/yum
- - sbin/sln
- - etc/ld.so.cache
- - var/cache/ldconfig
- - usr/share/backgrounds
-
-- name: Create needed directories
- file:
- path: "{{temp_root}}/{{item}}"
- state: directory
- with_items:
- - var/cache/yum
- - var/cache/ldconfig
-
-- name: Download filebeat
- get_url:
- url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-oss-{{ FILEBEAT_VERSION }}-linux-x86_64.tar.gz"
- dest: "{{ temp_root}}/opt/filebeat.tar.gz"
- mode: '0640'
-
-- name: Unarchive filebeat
- unarchive:
- src: "{{ temp_root}}/opt/filebeat.tar.gz"
- dest: "{{ temp_root}}/opt/"
- remote_src: yes
-
-- name: Delete filebeat archive
- file:
- path: "{{ item }}"
- state: absent
- with_items:
- - "{{ temp_root}}/opt/filebeat.tar.gz"
-
-- name: move filebeat directory to /opt/filebeat
- command: "mv {{ temp_root}}/opt/filebeat-{{ FILEBEAT_VERSION }}-linux-x86_64 {{ temp_root}}/opt/filebeat"
-
-- name: Import image in docker
- shell: tar --numeric-owner -c -C {{temp_root }} . | docker import - {{repo}}/centos:{{version}}{{suffix}}
-
-- name: Remove temp directory
- file:
- path: "{{temp_root}}"
- state: absent
-
+#- name: Skip if image exists
+# meta: end_play
+# when: centosimg.images | length != 0
+
+# tags:
+# - start
+
+#- name: Assert CentOS image
+# assert:
+# that: centosimg.images | length == 0
+# fail_msg: "CentOS image already exists"
+
+- name: Build CentOS image
+ when: centosimg.images | length == 0
+ block:
+ - name: Create etc tree in build directory
+ file:
+ path: '{{ temp_root}}/{{ item.path }}'
+ state: directory
+ mode: '{{ item.mode }}'
+ with_filetree: templates/etcroot/
+ when: item.state == 'directory'
+
+ - name: Populate etc tree in build directory
+ template:
+ src: '{{ item.src }}'
+ dest: '{{ temp_root}}/{{ item.path }}'
+ force: yes
+ with_filetree: templates/etcroot
+ when: item.state == 'file'
+
+ - name: Create dev tree in build directory
+ command: mknod -m {{ item.mode }} {{ item.dev }} {{ item.type }} {{ item.major }} {{ item.minor }}
+ args:
+ creates: "{{ item.dev }}"
+ with_items:
+ - { mode: 600, dev: "{{temp_root}}/dev/console", type: c, major: 5, minor: 1 }
+ - { mode: 600, dev: "{{temp_root}}/dev/initctl", type: p, major: '', minor: '' }
+ - { mode: 666, dev: "{{temp_root}}/dev/full", type: c, major: 1, minor: 7 }
+ - { mode: 666, dev: "{{temp_root}}/dev/null", type: c, major: 1, minor: 3 }
+ - { mode: 666, dev: "{{temp_root}}/dev/ptmx", type: c, major: 5, minor: 2 }
+ - { mode: 666, dev: "{{temp_root}}/dev/random", type: c, major: 1, minor: 8 }
+ - { mode: 666, dev: "{{temp_root}}/dev/tty", type: c, major: 5, minor: 0 }
+ - { mode: 666, dev: "{{temp_root}}/dev/tty0", type: c, major: 4, minor: 0 }
+ - { mode: 666, dev: "{{temp_root}}/dev/urandom", type: c, major: 1, minor: 9 }
+ - { mode: 666, dev: "{{temp_root}}/dev/zero", type: c, major: 1, minor: 5 }
+
+ - name: Install centos-release in build directory
+ yum:
+ installroot: "{{ temp_root}}"
+ name: centos-release
+ state: present
+
+ - name: Install Core CentOS in build directory
+ yum:
+ installroot: "{{ temp_root}}"
+ name:
+ - "@Core"
+ - yum-plugin-ovl.noarch
+ - epel-release
+ state: present
+
+ - name: Clean yum cache
+ command: 'yum --installroot="{{ temp_root}}" -y clean all'
+
+ - name: Remove unneeded directories
+ file:
+ path: "{{temp_root}}/{{item}}"
+ state: absent
+ with_items:
+ - usr/share/cracklib
+ - var/cache/yum
+ - sbin/sln
+ - etc/ld.so.cache
+ - var/cache/ldconfig
+ - usr/share/backgrounds
+
+ - name: Create needed directories
+ file:
+ path: "{{temp_root}}/{{item}}"
+ state: directory
+ with_items:
+ - var/cache/yum
+ - var/cache/ldconfig
+
+ - name: Import image in docker
+ shell: tar --numeric-owner -c -C {{temp_root }} . | docker import - {{repo}}/centos:{{version}}{{suffix}}
+
+ - name: Remove temp directory
+ file:
+ path: "{{temp_root}}"
+ state: absent
+
diff --git a/roles/build/tasks/main.yml b/roles/build/tasks/main.yml
index eee4ba0ee19f8eda382bed26c1e9dacf11706469..223766f87e2d7d1ae88de3d70dd7810a0dbdf091 100644
--- a/roles/build/tasks/main.yml
+++ b/roles/build/tasks/main.yml
@@ -2,45 +2,19 @@
- assert:
that:
- - "'CHANGE_ME' not in soctoolsproxy"
+ - "'CHANGE_ME' not in dslproxy"
fail_msg: "Review *all* settings in group_vars/all/main.yml"
- include: centos.yml
-
-- name: Create main build dir
- file:
- path: "{{docker_build_dir}}"
- state: directory
-
-- name: Create build dir
- file:
- path: "{{docker_build_dir}}/{{item}}"
- state: directory
- with_items: "{{services}}"
-
-- name: Configure the Dockerfile
- template:
- src: "{{item}}/Dockerfile.j2"
- dest: "{{docker_build_dir}}/{{item}}/Dockerfile"
- with_items: "{{services}}"
-
-- name: Copy thehive_button to build path
- copy:
- src: "{{role_path}}/templates/odfekibana/thehive_button"
- dest: "{{docker_build_dir}}/odfekibana/"
-
-- name: Copy keycloak-tools to build path
- copy:
- src: "{{role_path}}/templates/keycloak/keycloak-tools"
- dest: "{{docker_build_dir}}/keycloak/"
-
-- name: Copy build files
- copy:
- src: "files/{{item}}/"
- dest: "{{docker_build_dir}}/{{item}}/"
- with_items: "{{services}}"
- ignore_errors: yes
-
-- name: Build image
- command: docker build -t {{repo}}/{{item}}:{{version}}{{suffix}} -f {{docker_build_dir}}/{{item}}/Dockerfile {{docker_build_dir}}/{{item}}
- with_items: "{{services}}"
+- include: mysql.yml
+- include: haproxy.yml
+- include: openjdk.yml
+- include: zookeeper.yml
+- include: nifi.yml
+- include: odfees.yml
+- include: odfekibana.yml
+- include: keycloak.yml
+- include: misp.yml
+- include: cassandra.yml
+- include: thehive.yml
+- include: cortex.yml
diff --git a/roles/build/templates/cassandra/Dockerfile.j2 b/roles/build/templates/cassandra/Dockerfile.j2
index f5d2a601c1526e9cf119c01bea5e36c2edf8223d..94b0ca08c01de7c4b225bdcb81c7a35c2ffb33ff 100644
--- a/roles/build/templates/cassandra/Dockerfile.j2
+++ b/roles/build/templates/cassandra/Dockerfile.j2
@@ -1,20 +1,35 @@
FROM {{repo}}/openjdk:{{version}}{{suffix}}
USER root
+#COPY cassandra.repo /etc/yum.repos.d/cassandra.repo
+#COPY supervisord.conf /etc/supervisord.conf
+#COPY start.sh /start.sh
RUN echo "[cassandra]" > /etc/yum.repos.d/cassandra.repo && \
echo "name=Apache Cassandra" >> /etc/yum.repos.d/cassandra.repo && \
echo "baseurl=https://downloads.apache.org/cassandra/redhat/311x/" >> /etc/yum.repos.d/cassandra.repo && \
echo "gpgcheck=1" >> /etc/yum.repos.d/cassandra.repo && \
echo "repo_gpgcheck=1" >> /etc/yum.repos.d/cassandra.repo && \
echo "gpgkey=https://downloads.apache.org/cassandra/KEYS" >> /etc/yum.repos.d/cassandra.repo && \
+ echo '#!/bin/bash' > /start.sh && \
+ echo 'export CASSANDRA_HOME=/usr/share/cassandra' >> /start.sh && \
+ echo 'export CASSANDRA_CONF=$CASSANDRA_HOME/conf' >> /start.sh && \
+ echo 'export CASSANDRA_INCLUDE=$CASSANDRA_HOME/cassandra.in.sh' >> /start.sh && \
+ echo 'log_file=/var/log/cassandra/cassandra.log' >> /start.sh && \
+ echo 'pid_file=/var/run/cassandra/cassandra.pid' >> /start.sh && \
+ echo 'lock_file=/var/lock/subsys/cassandra' >> /start.sh && \
+ echo 'CASSANDRA_PROG=/usr/sbin/cassandra' >> /start.sh && \
+ echo '' >> /start.sh && \
+ echo '$CASSANDRA_PROG -p $pid_file > $log_file 2>&1' >> /start.sh && \
yum install -y epel-release && \
- yum install -y cassandra supervisor rsync && \
+ yum install -y cassandra supervisor && \
mkdir /usr/share/cassandra/conf && \
cp -a /etc/cassandra/conf/* /usr/share/cassandra/conf && \
chown -R cassandra:cassandra /usr/share/cassandra && \
chown -R cassandra:cassandra /var/lib/cassandra && \
sed -i -e 's,/etc/cassandra,/usr/share/cassandra,g' /usr/share/cassandra/cassandra.in.sh && \
+ chmod a+x /start.sh && \
yum -y clean all
-COPY cassandrasupervisord.conf /etc/supervisord.conf
EXPOSE 7000 9042
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+#ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+USER cassandra
+# ENTRYPOINT ["/start.sh"]
diff --git a/roles/build/templates/cortex/Dockerfile.j2 b/roles/build/templates/cortex/Dockerfile.j2
index ceeb6a59132f30e1ca21978a4ee873b97f05e2d3..d56dbf02c14be1e71860da43814eddc904aa7177 100644
--- a/roles/build/templates/cortex/Dockerfile.j2
+++ b/roles/build/templates/cortex/Dockerfile.j2
@@ -1,6 +1,9 @@
FROM {{repo}}/openjdk:{{version}}{{suffix}}
USER root
+#COPY thehive.repo /etc/yum.repos.d/thehive.repo
+#COPY supervisord.conf /etc/supervisord.conf
+#COPY start.sh /start.sh
RUN echo "[thehive-project]" > /etc/yum.repos.d/thehive.repo && \
echo "enabled=1" >> /etc/yum.repos.d/thehive.repo && \
echo "priority=1" >> /etc/yum.repos.d/thehive.repo && \
@@ -10,7 +13,7 @@ RUN echo "[thehive-project]" > /etc/yum.repos.d/thehive.repo && \
yum install -y epel-release && \
rpm --import https://raw.githubusercontent.com/TheHive-Project/TheHive/master/PGP-PUBLIC-KEY && \
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
- yum install -y cortex supervisor rsync daemonize vim net-tools telnet htop python3-pip.noarch git gcc python3-devel.x86_64 ssdeep-devel.x86_64 python3-wheel.noarch libexif-devel.x86_64 libexif.x86_64 perl-Image-ExifTool.noarch gcc-c++ whois && \
+ yum install -y cortex supervisor daemonize vim net-tools telnet htop python3-pip.noarch git gcc python3-devel.x86_64 ssdeep-devel.x86_64 python3-wheel.noarch libexif-devel.x86_64 libexif.x86_64 perl-Image-ExifTool.noarch gcc-c++ whois && \
rpm -Uvh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-6.8.13.rpm && \
chown -R elasticsearch:elasticsearch /etc/elasticsearch && \
mkdir -p /home/cortex && \
@@ -24,5 +27,6 @@ RUN echo "[thehive-project]" > /etc/yum.repos.d/thehive.repo && \
for I in responders/*/requirements.txt; do LC_ALL=en_US.UTF-8 pip3 install --no-cache-dir -U -r $I || true; done && \
yum -y clean all
EXPOSE 9001
-COPY cortexsupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+#ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+USER cortex
+# ENTRYPOINT ["/start.sh"]
diff --git a/roles/build/templates/cortex/application.conf b/roles/build/templates/cortex/application.conf
index afe42a610e0b4865af6e2d382cc88e46d7943882..0e28b4d0d71643d558b0d014be354985a02d19c7 100644
--- a/roles/build/templates/cortex/application.conf
+++ b/roles/build/templates/cortex/application.conf
@@ -17,7 +17,7 @@ search {
index = cortex3
# ElasticSearch instance address.
# For cluster, join address:port with ',': "http://ip1:9200,ip2:9200,ip3:9200"
- uri = "http://soctools-elastic:9200"
+ uri = "http://dsoclab-elastic:9200"
## Advanced configuration
# Scroll keepalive.
diff --git a/roles/build/templates/haproxy/Dockerfile.j2 b/roles/build/templates/haproxy/Dockerfile.j2
index 6c34d74ce038ef6feaa1bde99b2d823a30a830a9..d9f84c4c3ec60e5593ab4a3ccffee1660585260e 100644
--- a/roles/build/templates/haproxy/Dockerfile.j2
+++ b/roles/build/templates/haproxy/Dockerfile.j2
@@ -24,8 +24,6 @@ RUN \
iptables \
pcre2-devel \
daemonize \
- supervisor \
- rsync \
pth-devel && \
`# Install newest openssl...` \
wget -O /tmp/openssl.tgz https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz && \
@@ -64,5 +62,10 @@ RUN \
&& cp -R /usr/src/haproxy/examples/errorfiles /usr/local/etc/haproxy/errors \
&& rm -rf /usr/src/haproxy
-COPY haproxysupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+ENTRYPOINT ["/bin/bash"]
+
+# https://www.haproxy.org/download/1.8/doc/management.txt
+# "4. Stopping and restarting HAProxy"
+# "when the SIGTERM signal is sent to the haproxy process, it immediately quits and all established connections are closed"
+# "graceful stop is triggered when the SIGUSR1 signal is sent to the haproxy process"
+STOPSIGNAL SIGUSR1
diff --git a/roles/build/templates/keycloak/Dockerfile.j2 b/roles/build/templates/keycloak/Dockerfile.j2
index 561a6466f121ee2462dab388ed99a580f16ba39c..d8c3b6b46b96dac244154889943bb2aad256605c 100644
--- a/roles/build/templates/keycloak/Dockerfile.j2
+++ b/roles/build/templates/keycloak/Dockerfile.j2
@@ -11,7 +11,7 @@ USER root
#ADD /{{role_path}}/templates/keycloak/keycloak-tools /opt/jboss/tools
ADD keycloak-tools /opt/jboss/tools
#ADD ../templates/keycloak/keycloak-tools /opt/jboss/tools
-RUN yum -y install openssl supervisor rsync && yum -y clean all && \
+RUN yum -y install openssl && yum -y clean all && \
mkdir -p /opt/jboss/ && cd /opt/jboss/ && \
curl -L $KEYCLOAK_DIST | tar zx && \
mv /opt/jboss/keycloak-* /opt/jboss/keycloak && \
@@ -27,7 +27,6 @@ RUN yum -y install openssl supervisor rsync && yum -y clean all && \
adduser -u 1000 -g 0 -d /opt/jboss jboss && \
chown -R jboss:root /opt/jboss && \
chmod -R g+rwX /opt/jboss && \
- chmod a+x /opt/jboss/tools/x509.sh && \
mkdir -p /etc/x509/{https,ca} && chown -R jboss:root /etc/x509/{https,ca}
ENV PATH="/opt/jboss/keycloak/bin:${PATH}"
@@ -37,8 +36,6 @@ WORKDIR /opt/jboss/keycloak
EXPOSE 8080
EXPOSE 8443
-RUN echo 'jboss ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-COPY keycloaksupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+USER jboss
+ENTRYPOINT ["/bin/bash"]
diff --git a/roles/build/templates/misp/Dockerfile.j2 b/roles/build/templates/misp/Dockerfile.j2
index 85c96482f222cafeaf587b157dfda1eb28c84a43..b99d2a84cffc07c99a2622c5bba30c3e479a7425 100644
--- a/roles/build/templates/misp/Dockerfile.j2
+++ b/roles/build/templates/misp/Dockerfile.j2
@@ -2,7 +2,7 @@ FROM {{repo}}/centos:{{version}}{{suffix}}
USER root
RUN yum install -y epel-release centos-release-scl scl-utils ; \
- yum install -y gcc git zip openssl supervisor rsync rh-git218 httpd24 mod_ssl mod_auth_openidc rh-redis32 libxslt-devel zlib-devel libcaca-devel ssdeep-devel rh-php72 rh-php72-php-fpm rh-php72-php-devel rh-php72-php-mysqlnd rh-php72-php-mbstring rh-php72-php-xml rh-php72-php-bcmath rh-php72-php-opcache rh-php72-php-gd mariadb devtoolset-7 make cmake3 cppcheck libcxx-devel gpgme-devel openjpeg-devel gcc gcc-c++ poppler-cpp-devel pkgconfig python-devel redhat-rpm-config rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel wget screen rh-python36-mod_wsgi postfix curl make cmake python3 python3-devel python3-pip python3-yara python3-wheel python3-redis python3-zmq python3-setuptools redis sudo vim zip sqlite moreutils rng-tools libxml2-devel libxslt-devel zlib-devel libpqxx openjpeg2-devel ssdeep-devel ruby asciidoctor tesseract ImageMagick poppler-cpp-devel python36-virtualenv opencv-devel zbar zbar-devel ; \
+ yum install -y gcc git zip openssl supervisor rh-git218 httpd24 mod_ssl mod_auth_openidc rh-redis32 libxslt-devel zlib-devel libcaca-devel ssdeep-devel rh-php72 rh-php72-php-fpm rh-php72-php-devel rh-php72-php-mysqlnd rh-php72-php-mbstring rh-php72-php-xml rh-php72-php-bcmath rh-php72-php-opcache rh-php72-php-gd mariadb devtoolset-7 make cmake3 cppcheck libcxx-devel gpgme-devel openjpeg-devel gcc gcc-c++ poppler-cpp-devel pkgconfig python-devel redhat-rpm-config rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel wget screen rh-python36-mod_wsgi postfix curl make cmake python3 python3-devel python3-pip python3-yara python3-wheel python3-redis python3-zmq python3-setuptools redis sudo vim zip sqlite moreutils rng-tools libxml2-devel libxslt-devel zlib-devel libpqxx openjpeg2-devel ssdeep-devel ruby asciidoctor tesseract ImageMagick poppler-cpp-devel python36-virtualenv opencv-devel zbar zbar-devel ; \
yum -y clean all ; \
sed -i "s/max_execution_time = 30/max_execution_time = 300/" /etc/opt/rh/rh-php72/php.ini ; \
sed -i "s/memory_limit = 128M/memory_limit = 2048M/" /etc/opt/rh/rh-php72/php.ini ; \
@@ -76,12 +76,9 @@ RUN chown -R apache:apache /var/www/MISP ; \
chmod -R g+ws /var/www/MISP/app/files ; \
chmod -R g+ws /var/www/MISP/app/files/scripts/tmp
-COPY misp_rh-php72-php-fpm /etc/logrotate.d/rh-php72-php-fpm
-
# 80/443 - MISP web server, 3306 - mysql, 6379 - redis, 6666 - MISP modules, 50000 - MISP ZeroMQ
EXPOSE 80 443 6443 6379 6666 50000
-ENV PATH "$PATH:/opt/rh/rh-php72/root/bin/"
-
COPY mispsupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+#ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+
diff --git a/roles/build/templates/mysql/Dockerfile.j2 b/roles/build/templates/mysql/Dockerfile.j2
index 393eb3f8a06e049cbdb9b180f0cd4f093e7e86ab..c7b7d23bf199b273f76110b320863916cf173c67 100644
--- a/roles/build/templates/mysql/Dockerfile.j2
+++ b/roles/build/templates/mysql/Dockerfile.j2
@@ -2,7 +2,7 @@ FROM {{repo}}/centos:{{version}}{{suffix}}
USER root
RUN yum -y update && yum install -y epel-release centos-release-scl scl-utils && \
- yum install -y rh-mariadb103 python36-PyMySQL MySQL-python supervisor rsync && \
+ yum install -y rh-mariadb103 python36-PyMySQL MySQL-python supervisor && \
/usr/bin/scl enable rh-mariadb103 -- /opt/rh/rh-mariadb103/root/usr/libexec/mysql-prepare-db-dir /var/opt/rh/rh-mariadb103/lib/mysql
RUN yum clean all
diff --git a/roles/build/templates/nifi/Dockerfile.j2 b/roles/build/templates/nifi/Dockerfile.j2
index 63c51747ea349daa1509f556ee4c323043635c3c..d3408b16d5a52ad7cb48463cd1f543d2ab25c327 100644
--- a/roles/build/templates/nifi/Dockerfile.j2
+++ b/roles/build/templates/nifi/Dockerfile.j2
@@ -44,8 +44,6 @@ RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group ${GID} | cut -d: -
&& chown -R nifi:nifi ${NIFI_BASE_DIR} \
&& yum -y install jq xmlstarlet procps-ng
-RUN echo 'nifi ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
USER nifi
# Download, validate, and expand Apache NiFi Toolkit binary.
@@ -96,8 +94,4 @@ WORKDIR ${NIFI_HOME}
# Also we need to use relative path, because the exec form does not invoke a command shell,
# thus normal shell processing does not happen:
# https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example
-USER root
-RUN yum install -y supervisor rsync
-RUN yum clean all
-COPY nifisupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+ENTRYPOINT ["/bin/bash"]
diff --git a/roles/build/templates/odfekibana/Dockerfile-kibana.j2 b/roles/build/templates/odfekibana/Dockerfile-kibana.j2
index 05a1c40b3ac3cb718e524cef5225102e7ece1e55..c443597ddd2a10457ce9163c7b4dd722f2d8790f 100644
--- a/roles/build/templates/odfekibana/Dockerfile-kibana.j2
+++ b/roles/build/templates/odfekibana/Dockerfile-kibana.j2
@@ -1,8 +1,5 @@
FROM {{repo}}/centos:{{version}}{{suffix}}
-RUN yum install -y supervisor rsync
-RUN yum clean all
-
ENV PATH="/usr/share/kibana/bin:${PATH}"
RUN groupadd -g 1000 kibana && \
@@ -15,9 +12,7 @@ RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
cp -a /etc/kibana/ /usr/share/kibana/config/ && \
chown -R kibana /usr/share/kibana/config/
-RUN echo 'kibana ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
EXPOSE 5601
-COPY kibanasupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+USER kibana
+ENTRYPOINT ["/bin/bash"]
diff --git a/roles/build/templates/thehive/Dockerfile.j2 b/roles/build/templates/thehive/Dockerfile.j2
index 870e3ac179c6ee643639f63d0b69eff9ed900f95..773c7c2c7846ade845264764c68e22f36a0cf957 100644
--- a/roles/build/templates/thehive/Dockerfile.j2
+++ b/roles/build/templates/thehive/Dockerfile.j2
@@ -1,6 +1,9 @@
FROM {{repo}}/openjdk:{{version}}{{suffix}}
USER root
+#COPY thehive.repo /etc/yum.repos.d/thehive.repo
+#COPY supervisord.conf /etc/supervisord.conf
+#COPY start.sh /start.sh
RUN echo "[thehive-project]" > /etc/yum.repos.d/thehive.repo && \
echo "enabled=1" >> /etc/yum.repos.d/thehive.repo && \
echo "priority=1" >> /etc/yum.repos.d/thehive.repo && \
@@ -9,12 +12,13 @@ RUN echo "[thehive-project]" > /etc/yum.repos.d/thehive.repo && \
echo "gpgcheck=1" >> /etc/yum.repos.d/thehive.repo && \
yum install -y epel-release && \
rpm --import https://raw.githubusercontent.com/TheHive-Project/TheHive/master/PGP-PUBLIC-KEY && \
- yum install -y thehive4 supervisor daemonize vim net-tools telnet htop rsync && \
+ yum install -y thehive4 supervisor daemonize vim net-tools telnet htop && \
mkdir -p /opt/thp_data/files/thehive && \
chown -R thehive:thehive /opt/thp_data/files/thehive && \
mkdir -p /home/thehive && \
chown -R thehive:thehive /home/thehive /etc/thehive && \
yum -y clean all
EXPOSE 9000
-COPY thehivesupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+#ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+USER thehive
+# ENTRYPOINT ["/start.sh"]
diff --git a/roles/build/templates/zookeeper/Dockerfile.j2 b/roles/build/templates/zookeeper/Dockerfile.j2
index 19ae977882802ec3725ce8f7875b4801786153cf..209069c8137b1ee53bac1fbecda2efc9f388e9e2 100644
--- a/roles/build/templates/zookeeper/Dockerfile.j2
+++ b/roles/build/templates/zookeeper/Dockerfile.j2
@@ -29,8 +29,6 @@ EXPOSE 2181 2888 3888
WORKDIR ${ZOOKEEPER_BASE_DIR}/zookeeper
-#ENTRYPOINT ["/opt/zookeeper/bin/zkServer.sh"]
-#CMD ["start-foreground"]
-RUN yum install supervisor rsync -y
-COPY zookeepersupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+ENTRYPOINT ["/opt/zookeeper/bin/zkServer.sh"]
+CMD ["start-foreground"]
+
diff --git a/roles/ca/tasks/main.yml b/roles/ca/tasks/main.yml
index e851761376265268c0ea7af70ae65d47d789a43f..6ca350a28574cb3e5fb0accbe8244420c8303614 100644
--- a/roles/ca/tasks/main.yml
+++ b/roles/ca/tasks/main.yml
@@ -1,18 +1,8 @@
---
-- name: Create secret directory
- file:
- path: "{{playbook_dir}}/{{item}}"
- state: directory
- loop:
- - secrets
- - secrets/certificates
- - secrets/tokens
- - secrets/passwords
-
- name: Check for existing CA folder
stat:
- path: "{{playbook_dir}}/secrets/CA"
+ path: roles/ca/files/CA
register: capath
- name: build ca root key and cert
@@ -24,19 +14,27 @@
environment:
EASYRSA_BATCH: 1
EASYRSA_REQ_CN: "{{ ca_cn }}"
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
when: not capath.stat.exists
+- name: Copy cert to truststore
+ copy:
+ src: roles/ca/files/CA/ca.crt
+ dest: "roles/ca/files/truststore/{{ ca_cn }}.crt"
+
- name: Remove previous truststore
file:
- path: '{{playbook_dir}}/secrets/CA/cacerts.jks'
+ path: roles/ca/files/truststore/cacerts.jks
state: absent
- name: Generate truststore
command: >
- docker run --rm -v {{playbook_dir}}/secrets/CA/:/opt/cafiles/:z
+ docker run --rm -v {{role_path}}/files/truststore/:/opt/cafiles/:z
"{{repo}}/openjdk:{{version}}{{suffix}}" keytool -import -noprompt -trustcacerts
- -alias "{{ ca_cn }}" -file "/opt/cafiles/ca.crt" -keystore /opt/cafiles/cacerts.jks -storepass "{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
+ -alias "{{item}}" -file "/opt/cafiles/{{item}}.crt" -keystore /opt/cafiles/cacerts.jks -storepass "{{tspass}}"
+ with_items:
+ - "{{ ca_cn }}"
+ #- GN43WP8T31_CA
- name: Check for existing host certificates
command: roles/ca/files/easyrsa/easyrsa show-cert {{item}}
@@ -49,17 +47,16 @@
- "{{ groups['thehive'] }}"
- "{{ groups['cortex'] }}"
- "{{ groups['haproxy'] }}"
- - "filebeat"
environment:
EASYRSA_BATCH: 1
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
register: hostcerts
ignore_errors: true
- name: Generate host certificates
command: >
roles/ca/files/easyrsa/easyrsa
- --subject-alt-name="DNS:{{item}},DNS:{{soctoolsproxy}}"
+ --subject-alt-name="DNS:{{item}},DNS:{{dslproxy}}"
build-serverClient-full {{item}} nopass
with_items:
- "{{ groups['nificontainers'] }}"
@@ -70,10 +67,9 @@
- "{{ groups['thehive'] }}"
- "{{ groups['cortex'] }}"
- "{{ groups['haproxy'] }}"
- - "filebeat"
environment:
EASYRSA_BATCH: 1
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
ignore_errors: true
loop_control:
index_var: my_idx
@@ -97,7 +93,7 @@
expect:
command: roles/ca/files/easyrsa/easyrsa export-p12 {{item}}
responses:
- Enter Export Password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
+ Enter Export Password: "{{kspass}}"
with_items:
- "{{ groups['nificontainers'] }}"
- "{{ groups['odfeescontainers'] }}"
@@ -108,7 +104,145 @@
- "{{ groups['mispcontainers'] }}"
environment:
EASYRSA_BATCH: 1
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
+
+- name: Copy nifi host certs to nifi role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.p12
+ dest: roles/nifi/files/{{item}}.p12
+ with_items:
+ - "{{ groups['nificontainers'] }}"
+
+- name: Copy odfees host certs to odfees role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.p12
+ dest: roles/odfees/files/{{item}}.p12
+ with_items:
+ - "{{ groups['odfeescontainers'] }}"
+
+- name: Copy odfekibana host p12 certs to odfekibana role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.p12
+ dest: roles/odfekibana/files/{{item}}.p12
+ with_items:
+ - "{{ groups['odfekibanacontainers'] }}"
+
+- name: Copy cortex host p12 certs to cortex role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.p12
+ dest: roles/cortex/files/{{item}}.p12
+ with_items:
+ - "{{ groups['cortex'] }}"
+
+- name: Copy odfekibana host certs to odfekibana role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/odfekibana/files/{{item}}.crt
+ with_items:
+ - "{{ groups['odfekibanacontainers'] }}"
+
+- name: Copy odfekibana host keys to odfekibana role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/odfekibana/files/{{item}}.key
+ with_items:
+ - "{{ groups['odfekibanacontainers'] }}"
+
+- name: Copy haproxy host cert to haproxy role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/haproxy/files/{{item}}.crt
+ with_items:
+ - "{{ groups['haproxy'] }}"
+
+- name: Copy haproxy host key to haproxy role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/haproxy/files/{{item}}.key
+ with_items:
+ - "{{ groups['haproxy'] }}"
+
+- name: Copy keycloak host certs to keycloak role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/keycloak/files/{{item}}.crt
+ with_items:
+ - "{{ groups['keycloakcontainers'] }}"
+
+- name: Copy keycloak host keys to keycloak role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/keycloak/files/{{item}}.key
+ with_items:
+ - "{{ groups['keycloakcontainers'] }}"
+
+- name: Copy misp host certs to misp role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/misp/files/{{item}}.crt
+ with_items:
+ - "{{ groups['mispcontainers'] }}"
+
+- name: Copy misp host keys to misp role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/misp/files/{{item}}.key
+ with_items:
+ - "{{ groups['mispcontainers'] }}"
+
+- name: Copy thehive host cert to thehive role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/thehive/files/{{item}}.crt
+ with_items:
+ - "{{ groups['thehive'] }}"
+
+- name: Copy thehive host key to thehive role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/thehive/files/{{item}}.key
+ with_items:
+ - "{{ groups['thehive'] }}"
+
+- name: Copy cortex host cert to cortex role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/cortex/files/{{item}}.crt
+ with_items:
+ - "{{ groups['cortex'] }}"
+
+- name: Copy cortex host key to cortex role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/cortex/files/{{item}}.key
+ with_items:
+ - "{{ groups['cortex'] }}"
+
+- name: Copy truststore to roles
+ copy:
+ src: roles/ca/files/truststore/cacerts.jks
+ dest: "roles/{{item}}/files/cacerts.jks"
+ with_items:
+ - nifi
+ - odfees
+ - odfekibana
+ - keycloak
+ - misp
+ - cortex
+ - thehive
+
+- name: Copy ca cert to roles
+ copy:
+ src: "roles/ca/files/truststore/{{ ca_cn }}.crt"
+ dest: "roles/{{item}}/files/{{ ca_cn }}.crt"
+ with_items:
+ - nifi
+ - odfees
+ - odfekibana
+ - keycloak
+ - misp
+ - thehive
+ - cortex
- name: Check for existing user certificates
command: roles/ca/files/easyrsa/easyrsa show-cert {{item.CN | regex_escape()}}
@@ -116,7 +250,7 @@
- "{{soctools_users}}"
environment:
EASYRSA_BATCH: 1
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
register: usercerts
ignore_errors: true
@@ -126,7 +260,7 @@
- "{{soctools_users}}"
environment:
EASYRSA_BATCH: 1
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
ignore_errors: true
loop_control:
index_var: my_idx
@@ -136,17 +270,24 @@
expect:
command: roles/ca/files/easyrsa/easyrsa export-p12 "{{item.CN}}"
responses:
- Enter Export Password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/{{item.CN}}')}}"
+ Enter Export Password: "{{item.password}}"
with_items:
- "{{soctools_users}}"
environment:
EASYRSA_BATCH: 1
- EASYRSA_PKI: "{{playbook_dir}}/secrets/CA"
+ EASYRSA_PKI: roles/ca/files/CA
+
+- name: Copy user certs to odfees
+ copy:
+ src: "roles/ca/files/CA/private/{{ item.CN }}.p12"
+ dest: "roles/odfees/files/{{ item.CN }}.p12"
+ with_items:
+ - "{{soctools_users}}"
-- name: Copy user certs to certificates
+- name: Copy user certs to odfekibana
copy:
- src: "{{playbook_dir}}/secrets/CA/private/{{ item.CN }}.p12"
- dest: "{{playbook_dir}}/secrets/certificates/{{ item.CN }}.p12"
+ src: "roles/ca/files/CA/private/{{ item.CN }}.p12"
+ dest: "roles/odfekibana/files/{{ item.CN }}.p12"
with_items:
- "{{soctools_users}}"
diff --git a/roles/cassandra/tasks/main.yml b/roles/cassandra/tasks/main.yml
index f26e13b0d2279484dea86aa46ee76cb1b8088865..7912910a2ebcd07a79d6649b7529fae5bde3a0cf 100644
--- a/roles/cassandra/tasks/main.yml
+++ b/roles/cassandra/tasks/main.yml
@@ -1,17 +1,28 @@
---
-- include: start.yml
+- name: Configure Cassandra
+ template:
+ src: cassandra.yaml.j2
+ dest: /usr/share/cassandra/conf/cassandra.yaml
tags:
- - start
-- include: stop.yml
+ - start
+
+- name: Start Cassandra
+ command: "/start.sh"
tags:
- - stop
- - stop-cassandra
-- include: update-config.yml
+ - start
+
+- name: Wait for Cassandra
+ wait_for:
+ host: "{{groups['cassandra'][0]}}"
+ port: 9042
+ state: started
+ delay: 5
tags:
- - update-config
- - update-cassandra-config
-- include: restart.yml
+ - start
+
+- name: Stop Cassandra
+ command: "pkill -SIGTERM -F /var/run/cassandra/cassandra.pid"
tags:
- - restart
- - restart-cassandra
+ - stop
+
diff --git a/roles/cortex/tasks/main.yml b/roles/cortex/tasks/main.yml
index 5216d2c717a180540106714d58ae7d21241e4554..06b263933c3516e73f091c9e27ed4503869807f3 100644
--- a/roles/cortex/tasks/main.yml
+++ b/roles/cortex/tasks/main.yml
@@ -1,17 +1,108 @@
---
-- include: start.yml
+- name: Copy cacert to ca-trust dir
+ remote_user: root
+ copy:
+ src: "files/{{ca_cn}}.crt"
+ dest: /etc/pki/ca-trust/source/anchors/ca.crt
tags:
- - start
-- include: stop.yml
+ - start
+ - startcortex
+
+- name: Install cacert to root truststore
+ remote_user: root
+ command: "update-ca-trust"
+ tags:
+ - start
+ - startcortex
+
+- name: Copy certificates in cortex conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "/etc/cortex/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.p12"
+ - "{{ inventory_hostname }}.crt"
+ - "{{ inventory_hostname }}.key"
+ - cacerts.jks
+ - "{{ca_cn}}.crt"
+ tags:
+ - start
+ - startcortex
+
+- name: Get openid authkey
+ set_fact:
+ cortexsecret: "{{lookup('file', 'files/cortexsecret',convert_data=False) | from_json }}"
+ tags:
+ - start
+
+- name: Configure embedded Elasticsearch 6
+ remote_user: root
+ template:
+ src: jvm.options.j2
+ dest: /etc/elasticsearch/jvm.options
+ tags:
+ - start
+ - startcortex
+
+- name: Start embedded Elasticsearch 6
+ remote_user: root
+ command: >
+ daemonize
+ -u elasticsearch
+ -c /usr/share/elasticsearch
+ -p /tmp/elasticsearch.pid
+ -o /tmp/elasticsearch-stdout.log
+ /usr/share/elasticsearch/bin/elasticsearch
tags:
- - stop
- - stop-cortex
-- include: update-config.yml
+ - start
+ - startcortex
+
+- name: Configure Cortex
+ template:
+ src: application.conf.j2
+ dest: /etc/cortex/application.conf
tags:
- - update-config
- - update-cortex-config
-- include: restart.yml
+ - start
+ - startcortex
+
+- name: Configure Cortex logging
+ copy:
+ src: logback.xml
+ dest: /etc/cortex/logback.xml
tags:
- - restart
- - restart-cortex
+ - start
+
+- name: Start Cortex
+ command: >
+ daemonize
+ -c /opt/cortex
+ -p /tmp/cortex.pid
+ -o /tmp/cortex-stdout.log
+ /opt/cortex/bin/cortex
+ -Dconfig.file=/etc/cortex/application.conf
+ -Dlogger.file=/etc/cortex/logback.xml
+ -J-Xms1g
+ -J-Xmx1g
+ -Dpidfile.path=/dev/null
+ tags:
+ - start
+ - startcortex
+
+- name: Wait for Cortex
+ wait_for:
+ host: "{{groups['cortex'][0]}}"
+ port: 9001
+ state: started
+ delay: 5
+ tags:
+ - start
+ - startcortex
+
+- name: Stop Cortex
+ command: "pkill -SIGTERM -F /tmp/cortex.pid"
+ tags:
+ - stop
+ - stopcortex
+
diff --git a/roles/cortex/templates/application.conf.j2 b/roles/cortex/templates/application.conf.j2
index e45e446edc48c43d6c4e2a6ae7f9aa3f9b00f879..6d6d09c6808a308f791e19af61928cff430c3bdb 100644
--- a/roles/cortex/templates/application.conf.j2
+++ b/roles/cortex/templates/application.conf.j2
@@ -6,7 +6,7 @@
#
# IMPORTANT: If you deploy your application to several instances, make
# sure to use the same key.
-play.http.secret.key="{{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_secret_key')}}"
+play.http.secret.key="{{cortex_secret_key}}"
## ElasticSearch
search {
@@ -34,18 +34,18 @@ search {
## ## Authentication configuration
## search.username = "cortex"
-## search.password = "{{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_odfe')}}"
+## search.password = "{{cortex_odfe_pass}}"
##
## ## SSL configuration
## search.keyStore {
-## path = "/etc/cortex/soctools-cortex.p12"
+## path = "/etc/cortex/dsoclab-cortex.p12"
## type = "PKCS12" # or PKCS12
-## password = "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
+## password = "{{kspass}}"
## }
## search.trustStore {
## path = "/etc/cortex/cacerts.jks"
## type = "JKS" # or PKCS12
-## password = "{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
+## password = "{{tspass}}"
## }
}
@@ -66,7 +66,7 @@ auth {
# the "ad" section below.
# - ldap : use LDAP to authenticate users. The associated configuration shall be done in the
# "ldap" section below.
- provider = [local]
+ provider = [local,oauth2]
ad {
# The Windows domain name in DNS format. This parameter is required if you do not use
@@ -108,6 +108,84 @@ auth {
# If 'true', use SSL to connect to the LDAP directory server.
#useSSL = true
}
+ oauth2 {
+ # URL of the authorization server
+ clientId = "dsoclab-cortex"
+ clientSecret = {{cortexsecret.value}}
+ redirectUri = "https://{{dslproxy}}:9001/api/ssoLogin"
+ responseType = "code"
+ grantType = "authorization_code"
+
+ # URL from where to get the access token
+ authorizationUrl = "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/auth"
+ authorizationHeader = "Bearer"
+ tokenUrl = "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/token"
+
+
+ # The endpoint from which to obtain user details using the OAuth token, after successful login
+ userUrl = "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/userinfo"
+ scope = "profile"
+ userIdField = "email"
+ #userUrl = "https://auth-site.com/api/User"
+ #scope = ["openid profile"]
+ }
+
+ ws.ssl.trustManager {
+ stores = [
+ {
+ type = "JKS" // JKS or PEM
+ path = "cacerts.jks"
+ password = "{{tspass}}"
+ }
+ ]
+ }
+
+
+ # Single-Sign On
+ sso {
+ # Autocreate user in database?
+ autocreate = true
+
+ # Autoupdate its profile and roles?
+ autoupdate = true
+
+ # Autologin user using SSO?
+ autologin = true
+
+ # Name of mapping class from user resource to backend user ('simple' or 'group')
+ #mapper = group
+ #mapper = simple
+ #attributes {
+ # login = "user"
+ # name = "name"
+ # groups = "groups"
+ # organization = "org"
+ #}
+# defaultRoles = ["read", "write", "admin"]
+# defaultOrganization = "uninett.no"
+ #defaultRoles = ["read"]
+ #defaultOrganization = "csirt"
+ #groups {
+ # # URL to retreive groups (leave empty if you are using OIDC)
+ # #url = "https://auth-site.com/api/Groups"
+ # # Group mappings, you can have multiple roles for each group: they are merged
+ # mappings {
+ # admin-profile-name = ["admin"]
+ # editor-profile-name = ["write"]
+ # reader-profile-name = ["read"]
+ # }
+ #}
+
+ mapper = simple
+ attributes {
+ login = "user"
+ name = "name"
+ roles = "roles"
+ organization = "org"
+ }
+ defaultRoles = ["read", "analyze"]
+ defaultOrganization = "uninett.no"
+ }
}
## ANALYZERS
diff --git a/roles/docker/tasks/cassandra.yml b/roles/docker/tasks/cassandra.yml
index b913425d7171f5bb30f9b5a2434b521a09879ae6..a26bd28c104bc70cc17916496a1da8b68d978cd9 100644
--- a/roles/docker/tasks/cassandra.yml
+++ b/roles/docker/tasks/cassandra.yml
@@ -10,6 +10,7 @@
networks_cli_compatible: yes
volumes:
- "{{cassandra_name}}:/var/lib/cassandra"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['cassandra'] }}"
tags:
diff --git a/roles/docker/tasks/cortex.yml b/roles/docker/tasks/cortex.yml
index 15a0732d1d90e4403f020e1edf868c34ab0d5dfc..c8d7b1ea2fd7564d5e4829620e994773168d11c7 100644
--- a/roles/docker/tasks/cortex.yml
+++ b/roles/docker/tasks/cortex.yml
@@ -12,6 +12,7 @@
# - "9001:9001"
volumes:
- "{{item}}:/var/lib/elasticsearch/"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['cortex'] }}"
tags:
diff --git a/roles/docker/tasks/haproxy.yml b/roles/docker/tasks/haproxy.yml
index b0180761f4f3c18982f21d9f5b8307066146daed..10dc08e09d929ab86254e4bb37b962da8880338c 100644
--- a/roles/docker/tasks/haproxy.yml
+++ b/roles/docker/tasks/haproxy.yml
@@ -15,10 +15,9 @@
- "9443:9443"
- "9200:9200"
- "7750:7750"
- - "5000-5099:5000-5099"
- - "6000-6099:6000-6099"
- "9000:9000"
- "9001:9001"
+ entrypoint: "/bin/bash"
interactive: "yes"
tags:
- start
diff --git a/roles/docker/tasks/keycloak.yml b/roles/docker/tasks/keycloak.yml
index 1fddf2a13aa6086a09ef7ad9c0f8ea1c7aa7b3b9..c910408f048a5fa76ace4a47ec1c2d32a314c21f 100644
--- a/roles/docker/tasks/keycloak.yml
+++ b/roles/docker/tasks/keycloak.yml
@@ -10,6 +10,7 @@
networks_cli_compatible: yes
published_ports:
- "12443:8443"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['keycloakcontainers'] }}"
tags:
diff --git a/roles/docker/tasks/misp.yml b/roles/docker/tasks/misp.yml
index f32e440eeff7e9d8c1406cf76bafde97aaba19d0..34a42ad3c033abf3c58a1664a92205b9cb76f69e 100644
--- a/roles/docker/tasks/misp.yml
+++ b/roles/docker/tasks/misp.yml
@@ -7,8 +7,9 @@
image: "{{ misp_img }}"
networks:
- name: "{{ soctools_netname}}"
- interactive: "yes"
networks_cli_compatible: yes
+ entrypoint: "/bin/bash"
+ interactive: "yes"
published_ports:
- "6443:6443"
tags:
diff --git a/roles/docker/tasks/nifi.yml b/roles/docker/tasks/nifi.yml
index e85fb61a10993c070180291d6c1ff8d06c74dece..7023444e9c0181dcde646dd0f1b4e55dc1f2b031 100644
--- a/roles/docker/tasks/nifi.yml
+++ b/roles/docker/tasks/nifi.yml
@@ -10,6 +10,7 @@
networks_cli_compatible: yes
volumes:
- "{{item}}:/opt/nifi/nifi-current/conf"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['nificontainers'] }}"
tags:
diff --git a/roles/docker/tasks/odfees.yml b/roles/docker/tasks/odfees.yml
index fa35ddd58f54e597b24c030636c34fb650f732b8..d54ebd59216d12b7c42e15005b3f6b15fb9affa9 100644
--- a/roles/docker/tasks/odfees.yml
+++ b/roles/docker/tasks/odfees.yml
@@ -10,6 +10,7 @@
networks_cli_compatible: yes
volumes:
- "{{item}}:/usr/share/elasticsearch/data"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['odfeescontainers'] }}"
tags:
diff --git a/roles/docker/tasks/odfekibana.yml b/roles/docker/tasks/odfekibana.yml
index a1c88a2932cd76350e3fedadbec1e2f5511d0748..c24611b8869d7b1163065721d600ae7318356ddc 100644
--- a/roles/docker/tasks/odfekibana.yml
+++ b/roles/docker/tasks/odfekibana.yml
@@ -10,6 +10,7 @@
networks_cli_compatible: yes
published_ports:
- "5601:5601"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['odfekibanacontainers'] }}"
tags:
diff --git a/roles/docker/tasks/thehive.yml b/roles/docker/tasks/thehive.yml
index 68fdd3c3f21e711ee957feee298b24544e9bbc66..30b11c82ba446fffadd57d3f4e102322e49bb0b8 100644
--- a/roles/docker/tasks/thehive.yml
+++ b/roles/docker/tasks/thehive.yml
@@ -10,10 +10,12 @@
networks_cli_compatible: yes
# published_ports:
# - "9000:9000"
+ entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['thehive'] }}"
tags:
- start
+ - thehivestart
- name: Disconnect thehive containers from network and remove
docker_container:
@@ -22,4 +24,4 @@
with_items: "{{ groups['thehive'] }}"
tags:
- stop
-
+ - thehivestop
diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml
index 00ff3573b0523cbc4c0f71d1514bc862117d7e6e..d8bcefb127b59568b20742ec123ca25130438ee6 100644
--- a/roles/haproxy/tasks/main.yml
+++ b/roles/haproxy/tasks/main.yml
@@ -1,17 +1,68 @@
---
+# tasks file for haproxy
-- include: start.yml
+- name: Copy haproxy configuration file
+ template:
+ src: haproxy.cfg.j2
+ dest: /usr/local/etc/haproxy/haproxy.cfg
tags:
- - start
-- include: stop.yml
+ - start
+
+- name: Create required directories
+ file:
+ path: "{{ item }}"
+ state: directory
+ mode: 0755
+ owner: root
+ group: root
+ with_items:
+ - /opt/haproxy
+ - /etc/ssl/haproxy
+ tags:
+ - start
+
+- name: Generate dhparam file for haproxy
+ shell: "openssl dhparam -out /usr/local/etc/haproxy/dhparam.pem 2048"
tags:
- - stop
- - stop-haproxy
-- include: update-config.yml
+ - start
+
+- name: Copy haproxy certificates
+ copy:
+ src: "{{ item }}"
+ dest: "/opt/haproxy/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.crt"
+ - "{{ inventory_hostname }}.key"
tags:
- - update-config
- - update-haproxy-config
-- include: restart.yml
+ - start
+
+- name: Combine crt and key for haproxy
+ assemble:
+ src: /opt/haproxy
+ dest: /etc/ssl/haproxy/{{ inventory_hostname }}.crt
+ owner: root
+ mode: 0600
tags:
- - restart
- - restart-haproxy
+ - start
+
+- name: Delete temporary files and directory
+ file:
+ path: "{{ item }}"
+ state: absent
+ with_items:
+ - /opt/haproxy/{{ inventory_hostname }}.crt
+ - /opt/haproxy/{{ inventory_hostname }}.key
+ - /opt/haproxy
+ tags:
+ - start
+
+- name: Start haproxy
+ shell: "daemonize -c / -p /haproxy.pid /usr/local/sbin/haproxy -f /usr/local/etc/haproxy/haproxy.cfg"
+ tags:
+ - start
+
+#- name: Stop haproxy
+# tags:
+# - stop
+
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2
index d566981a6c63627dddb93a94d562a497856c349e..e62740e78b24395594aeef2789b464eed20ecc7b 100644
--- a/roles/haproxy/templates/haproxy.cfg.j2
+++ b/roles/haproxy/templates/haproxy.cfg.j2
@@ -1,6 +1,6 @@
global
#quiet
- log 127.0.0.1:9000 local0
+ log stdout format raw local0
nbproc {{ HAPROXY_PROCESSES }}
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
@@ -22,7 +22,7 @@ listen stats
stats hide-version
stats uri /
stats realm HAProxy Statistics
- stats auth haproxy:{{lookup('password', '{{playbook_dir}}/secrets/passwords/haproxy_stats')}}
+ stats auth haproxy:{{ HAPROXY_STATS_PASS }}
listen nifiserv
bind *:9443 ssl crt /etc/ssl/haproxy alpn h2,http/1.1
@@ -31,7 +31,6 @@ listen nifiserv
fullconn 5000
balance source
option tcpka
- option httplog
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:9443 ssl check verify none
{% endfor %}
@@ -43,7 +42,6 @@ listen odfeserv
fullconn 5000
balance source
option tcpka
- option httplog
{% for odfehost in groups['odfeescontainers'] %}
server {{odfehost}} {{odfehost}}:9200 ssl check verify none
{% endfor %}
@@ -55,7 +53,6 @@ listen keycloakserv
fullconn 5000
balance source
option tcpka
- option httplog
{% for keycloakhost in groups['keycloakcontainers'] %}
server {{keycloakhost}} {{keycloakhost}}:8443 ssl check verify none
{% endfor %}
@@ -68,7 +65,6 @@ listen thehiveserv
balance source
option tcpka
option forwardfor
- option httplog
{% for thehivehost in groups['thehive'] %}
server {{thehivehost}} {{thehivehost}}:9000 check verify none
{% endfor %}
@@ -81,7 +77,6 @@ listen cortexserv
balance source
option tcpka
option forwardfor
- option httplog
{% for cortexhost in groups['cortex'] %}
server {{cortexhost}} {{cortexhost}}:9001 check verify none
{% endfor %}
@@ -95,7 +90,6 @@ listen nifiservtcp77{{port}}
balance source
option tcpka
option tcp-check
- option tcplog
tcp-check connect port 77{{port}}
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:77{{port}} check
@@ -112,7 +106,6 @@ listen nifiservtcp7771
balance source
option tcpka
option tcp-check
- option tcplog
tcp-check connect port 7771
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:7771 check
@@ -127,7 +120,6 @@ listen nifiservhttp{{port}}
fullconn 5000
balance source
option tcpka
- option httplog
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:{{port}} check
{% endfor %}
@@ -144,7 +136,6 @@ listen nifiservtcp{{port}}
balance source
option tcpka
option tcp-check
- option tcplog
tcp-check connect port {{port}}
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}} {{nifihost}}:{{port}} check
diff --git a/roles/keycloak/tasks/main.yml b/roles/keycloak/tasks/main.yml
index d7668ba48273ed7f841c5e4e868aaa4415817194..2bb6a62470c5e10d59be3a56109969f5fe802025 100644
--- a/roles/keycloak/tasks/main.yml
+++ b/roles/keycloak/tasks/main.yml
@@ -1,17 +1,97 @@
---
-- include: start.yml
- tags:
- - start
-- include: stop.yml
- tags:
- - stop
- - stop-keycloak
-- include: update-config.yml
- tags:
- - update-config
- - update-keycloak-config
-- include: restart.yml
- tags:
- - restart
- - restart-keycloak
+- name: Copy certificates in keycloak x509 conf dir
+ copy:
+ src: "{{ item.local }}"
+ dest: "{{ item.remote }}"
+ mode: "{{ item.mode }}"
+ with_items:
+ - local: "files/{{ inventory_hostname }}.crt"
+ remote: /etc/x509/https/tls.crt
+ mode: '0644'
+ - local: "files/{{ inventory_hostname }}.key"
+ remote: /etc/x509/https/tls.key
+ mode: '0600'
+ - local: "files/{{ ca_cn }}.crt"
+ remote: /etc/x509/ca/ca.crt
+ mode: '0644'
+ - local: "files/cacerts.jks"
+ remote: /opt/jboss/keycloak/cacerts.jks
+ mode: '0644'
+ tags:
+ - start
+ - startkeycloak
+
+- name: Generate Keycloak secure config
+ command: "/opt/jboss/tools/x509.sh"
+ environment:
+ X509_CA_BUNDLE: "/etc/x509/ca/ca.crt"
+ tags:
+ - start
+ - startkeycloak
+
+- name: Set admin password
+ command: /opt/jboss/keycloak/bin/add-user-keycloak.sh --user "admin" --password "{{keycloak_adminpass}}"
+ ignore_errors: yes
+ tags:
+ - start
+ - startkeycloak
+
+- name: Configure Keycloak start script
+ template:
+ src: "{{item}}.j2"
+ dest: "/opt/jboss/tools/{{item}}"
+ mode: 0750
+ with_items:
+ - startkeycloak.sh
+ - initkeycloakrealm.sh
+ tags:
+ - start
+ - startkeycloak
+
+
+- name: Start Keycloak IdP
+ command: /opt/jboss/tools/startkeycloak.sh
+ tags:
+ - start
+ - startkeycloak
+
+- name: Wait for Keycloak
+ wait_for:
+ host: "{{groups['keycloakcontainers'][0]}}"
+ port: 8443
+ state: started
+ delay: 5
+ tags:
+ - start
+ - startkeycloak
+
+- name: Initialize Keycloak realm
+ command: /opt/jboss/tools/initkeycloakrealm.sh
+ tags:
+ - start
+ - startkeycloak
+
+- name: Copy secrets from Keycloak
+ fetch:
+ src: "{{ item.remote }}"
+ dest: "{{ item.local }}"
+ flat: yes
+ with_items:
+ - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/nifisecret"
+ local: "roles/nifi/files/nifisecret"
+ - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/kibanasecret"
+ local: "roles/odfekibana/files/kibanasecret"
+ - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/thehivesecret"
+ local: "roles/thehive/files/thehivesecret"
+ - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/cortexsecret"
+ local: "roles/cortex/files/cortexsecret"
+ tags:
+ - start
+ - startkeycloak
+
+- name: Stop Keycloak
+ command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+ tags:
+ - stop
+ - stopkeycloak
diff --git a/roles/keycloak/templates/initkeycloakrealm.sh.j2 b/roles/keycloak/templates/initkeycloakrealm.sh.j2
index 3d790f2eb02af9dbb6644d50795d77788f8c0ba4..d6fc9463d0bddd7f8f31587b6ecf7019d3f69428 100644
--- a/roles/keycloak/templates/initkeycloakrealm.sh.j2
+++ b/roles/keycloak/templates/initkeycloakrealm.sh.j2
@@ -5,8 +5,8 @@ exec 7>&2
exec > /opt/jboss/keycloak/initkeycloak.log 2>&1
-kcadm.sh config truststore --trustpass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} /opt/jboss/keycloak/cacerts.jks
-kcadm.sh config credentials --server https://{{groups['keycloakcontainers'][0]}}:8443/auth --realm master --user admin --password "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keykloak_admin')}}"
+kcadm.sh config truststore --trustpass {{tspass}} /opt/jboss/keycloak/cacerts.jks
+kcadm.sh config credentials --server https://{{groups['keycloakcontainers'][0]}}:8443/auth --realm master --user admin --password {{keycloak_adminpass}}
kcadm.sh create realms -b '{ "enabled": "true", "id": "{{openid_realm}}", "realm": "{{openid_realm}}"}'
kcadm.sh create realms/{{openid_realm}}/authentication/flows/browser/copy -b '{ "id": "browser-x509", "newName": "X.509 Browser" }'
BROWSERFORM=$(kcadm.sh create realms/{{openid_realm}}/authentication/flows/X.509%20Browser/executions/execution -i -b '{ "provider": "auth-x509-client-username-form" }')
@@ -18,18 +18,22 @@ kcadm.sh create realms/{{openid_realm}}/groups -b '{"name":"GN43WP8T31"}'
{% for user in soctools_users %}
kcadm.sh create realms/{{openid_realm}}/users -b '{"enabled":true,"attributes":{"DN": ["{{user.DN}}"],"CN": ["{{user.CN}}"]},"username":"{{user.username}}","emailVerified":"","email":"{{user.email}}","firstName":"{{user.firstname}}","lastName":"{{user.lastname}}","groups": ["/GN43WP8T31"] }'
-kcadm.sh set-password -r {{openid_realm}} --username {{user.username}} --new-password {{lookup('password', '{{playbook_dir}}/secrets/passwords/'+user.CN)}}
+kcadm.sh set-password -r {{openid_realm}} --username {{user.username}} --new-password {{user.password}}
{% endfor %}
-NIFICLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-nifi","protocol":"openid-connect","clientAuthenticatorType": "client-secret","redirectUris": ["https://{{soctoolsproxy}}:9443/*" ],"webOrigins": [], "publicClient": false }')
+NIFICLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-nifi","protocol":"openid-connect","clientAuthenticatorType": "client-secret","redirectUris": ["https://{{dslproxy}}:9443/*" ],"webOrigins": [], "publicClient": false }')
kcadm.sh create realms/{{openid_realm}}/clients/${NIFICLIENT}/protocol-mappers/models -b '{"protocol":"openid-connect","config":{"id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","multivalued":"","aggregate.attrs":"","user.attribute":"DN","claim.name":"DN","jsonType.label":"String"},"name":"SendDN","protocolMapper":"oidc-usermodel-attribute-mapper"}'
kcadm.sh get realms/{{openid_realm}}/clients/${NIFICLIENT}/client-secret --fields value > /opt/jboss/keycloak/nifisecret
-KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-kibana","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:5601","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:5601", "https://{{soctoolsproxy}}:5601/auth/openid/login", "https://{{soctoolsproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }')
+KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-kibana","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{dslproxy}}:5601","adminUrl": "","redirectUris": ["https://{{dslproxy}}:5601", "https://{{dslproxy}}:5601/auth/openid/login", "https://{{dslproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }')
kcadm.sh get realms/{{openid_realm}}/clients/${KIBANACLIENT}/client-secret --fields value > /opt/jboss/keycloak/kibanasecret
-MISPCLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-misp","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:6443","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:6443/users/login/keycloak"],"webOrigins": [], "publicClient": false }')
-kcadm.sh get realms/{{openid_realm}}/clients/${MISPCLIENT}/client-secret --fields value > /opt/jboss/keycloak/mispsecret
+THEHIVECLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-thehive","protocol":"openid-connect","clientAuthenticatorType": "client-secret","adminUrl": "","redirectUris": ["https://{{dslproxy}}:9000/api/ssoLogin"],"webOrigins": [], "publicClient": false }')
+kcadm.sh get realms/{{openid_realm}}/clients/${THEHIVECLIENT}/client-secret --fields value > /opt/jboss/keycloak/thehivesecret
+
+CORTEXCLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-cortex","protocol":"openid-connect","clientAuthenticatorType": "client-secret","adminUrl": "","redirectUris": ["https://{{dslproxy}}:9001/api/ssoLogin"],"webOrigins": [], "publicClient": false }')
+kcadm.sh get realms/{{openid_realm}}/clients/${CORTEXCLIENT}/client-secret --fields value > /opt/jboss/keycloak/cortexsecret
+
kcadm.sh config truststore --delete
diff --git a/roles/misp/tasks/main.yml b/roles/misp/tasks/main.yml
index 9b53c6c2b6dd1a2bc3207018e2524abfc390af9f..826e9dce14466e41450089a3720f28d4a4e54573 100644
--- a/roles/misp/tasks/main.yml
+++ b/roles/misp/tasks/main.yml
@@ -1,19 +1,59 @@
---
-- include: start.yml
+- name: Copy certificates in apache cert dir
+ copy:
+ src: "{{ item.local }}"
+ dest: "{{ item.remote }}"
+ mode: "{{ item.mode}}"
+ with_items:
+ - local: "files/{{ inventory_hostname }}.crt"
+ remote: /etc/ssl/certs/misp.crt
+ mode: '0644'
+ - local: "files/{{ inventory_hostname }}.key"
+ remote: /etc/ssl/certs/misp.key
+ mode: '0600'
+ - local: "files/{{ ca_cn }}.crt"
+ remote: /etc/ssl/certs/ca.crt
+ mode: '0644'
tags:
- - start
-- include: config.yml
+ - start
+
+- name: Configure Apache web server for misp
+ template:
+ src: misp.conf.j2
+ dest: /etc/httpd/conf.d/misp.conf
+ tags:
+ - start
+
+- name: Configure MISP database access
+ template:
+ src: database.php.j2
+ dest: /var/www/MISP/app/Config/database.php
+ tags:
+ - start
+
+- name: Configure MISP app
+ template:
+ src: config.php.j2
+ dest: /var/www/MISP/app/Config/config.php
tags:
- - start
- - config
-- include: stop.yml
- tags: stop
-- include: update-config.yml
+ - start
+
+- name: Configure MISP database initialization script
+ template:
+ src: checkdb.sh.j2
+ dest: /var/www/MISP/checkdb.sh
+ mode: '0700'
tags:
- - update-config
- - update-misp-config
-- include: restart.yml
+ - start
+
+- name: Check if database is initialized
+ command: /var/www/MISP/checkdb.sh
tags:
- - restart
- - restart-misp
+ - start
+
+- name: Start MISP
+ command: "/usr/bin/supervisord -c /etc/supervisord.conf"
+ tags:
+ - start
+
diff --git a/roles/misp/templates/checkdb.sh.j2 b/roles/misp/templates/checkdb.sh.j2
index c8eb4abac0fefe53b36d5ac6fb5e588e96a03054..5aba69f866bb0343a8abbbadd3e6606213574123 100644
--- a/roles/misp/templates/checkdb.sh.j2
+++ b/roles/misp/templates/checkdb.sh.j2
@@ -1,5 +1,5 @@
#!/bin/bash -x
-MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{mysql_name}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}})
+MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}})
if [ ${MISPINIT} == "0" ]; then
- cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{mysql_name}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}}
+ cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}}
fi
diff --git a/roles/misp/templates/database.php.j2 b/roles/misp/templates/database.php.j2
index 867e5fb9e876d23e8d6552eab177a1e5121c8eff..0cf1934f76488c257a3f68c2a4cf97ae58a8dead 100755
--- a/roles/misp/templates/database.php.j2
+++ b/roles/misp/templates/database.php.j2
@@ -63,11 +63,11 @@ class DATABASE_CONFIG {
'datasource' => 'Database/Mysql',
//'datasource' => 'Database/Postgres',
'persistent' => false,
- 'host' => '{{mysql_name}}',
+ 'host' => '{{groups['mysql'][0]}}',
'login' => '{{misp_dbuser}}',
'port' => 3306, // MySQL & MariaDB
//'port' => 5432, // PostgreSQL
- 'password' => '{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}}',
+ 'password' => '{{misp_dbpass}}',
'database' => '{{misp_dbname}}',
'prefix' => '',
'encoding' => 'utf8',
diff --git a/roles/misp/templates/misp.conf.j2 b/roles/misp/templates/misp.conf.j2
index 2ca05216dba90d63323d1362325837dd83824fbd..3a9cb85aa753ba49e5cdeec63daf64a76f46e340 100644
--- a/roles/misp/templates/misp.conf.j2
+++ b/roles/misp/templates/misp.conf.j2
@@ -1,7 +1,7 @@
Listen 6443 https
<VirtualHost *:6443>
-ServerAdmin admin@{{soctoolsproxy}}
-ServerName {{soctoolsproxy}}
+ServerAdmin admin@{{dslproxy}}
+ServerName {{dslproxy}}
DocumentRoot /var/www/MISP/app/webroot
<Directory /var/www/MISP/app/webroot>
Options -Indexes
@@ -10,25 +10,15 @@ AllowOverride all
DirectoryIndex index.php
+# ProxyPassMatch ^/info$ fcgi://127.0.0.1:9000/var/www/MISP/app/webroot/info.php
ProxyPassMatch ^/(.*\.php(/.*)?)$ fcgi://127.0.0.1:9000/var/www/MISP/app/webroot/$1
-SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1
-
-OIDCCryptoPassphrase {{lookup('password', '{{playbook_dir}}/secrets/passwords/misp_crypto')}}
-OIDCProviderMetadataURL https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
-OIDCRedirectURI https://{{soctoolsproxy}}:6443/users/login/keycloak
-OIDCClientID soctools-misp
-OIDCScope "openid profile"
-OIDCClientSecret {{mispsecret.value}}
-OIDCRemoteUserClaim email
-OIDCProviderTokenEndpointAuth client_secret_basic
-
-<Location /users/login>
- AuthType openid-connect
- Require valid-user
- RequestHeader set X-Remote-User %{OIDC_CLAIM_email}e
-</Location>
-
+#<FilesMatch "\.php$">
+# SetHandler "proxy:fcgi://127.0.0.1:9000"
+# ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
+# AddHandler php72-fcgi .php
+# Action php72-fcgi /cgi-bin/php72.fcgi
+#</FilesMatch>
SSLEngine On
SSLCertificateFile /etc/ssl/certs/misp.crt
diff --git a/roles/misp/templates/mysql_secure.sql.j2 b/roles/misp/templates/mysql_secure.sql.j2
index dd8ffd5d96d8dcdc08ca9efca5b2f66f3fa82dc9..7b8dd283a33da8aab7f443985ea9f5f97fb6f5d6 100644
--- a/roles/misp/templates/mysql_secure.sql.j2
+++ b/roles/misp/templates/mysql_secure.sql.j2
@@ -1,4 +1,4 @@
-UPDATE mysql.user SET Password=PASSWORD('{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}') WHERE User='root';
+UPDATE mysql.user SET Password=PASSWORD('{{mysql_dbrootpass}}') WHERE User='root';
DELETE FROM mysql.user WHERE User='';
DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
DROP DATABASE IF EXISTS test;
@@ -6,7 +6,7 @@ DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%';
CREATE DATABASE {{misp_dbname}};
{% for misp_host in groups['mispcontainers'] %}
-GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}}';
+GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{misp_dbpass}}';
GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}';
{% endfor %}
diff --git a/roles/mysql/tasks/main.yml b/roles/mysql/tasks/main.yml
index 69395fe3e0562adca1f273db7d17f8e3b1451ee6..f915611043cb4f4d1aa7aa9b2d9c4b87539ad4c5 100644
--- a/roles/mysql/tasks/main.yml
+++ b/roles/mysql/tasks/main.yml
@@ -1,20 +1,4 @@
---
- include: secure.yml
- tags:
- - start
- include: misp.yml
- tags:
- - start
-- include: stop.yml
- tags:
- - stop
- - stop-mysql
-- include: update-config.yml
- tags:
- - update-config
- - update-mysql-config
-- include: restart.yml
- tags:
- - restart
- - restart-mysql
diff --git a/roles/mysql/tasks/misp.yml b/roles/mysql/tasks/misp.yml
index c541e99243876ad9739ec4b52bc946b481ef6274..60e1d0a082f9036afb9295a5b9f8ad2e3713f843 100644
--- a/roles/mysql/tasks/misp.yml
+++ b/roles/mysql/tasks/misp.yml
@@ -4,26 +4,23 @@
mysql_db:
name: "{{misp_dbname}}"
state: present
+ tags:
+ - start
- name: Create misp user
mysql_user:
name: "{{misp_dbuser}}"
- #host: "{{item}}.{{soctools_netname}}"
- host: "%"
- password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}}"
+ host: "{{item}}.{{soctools_netname}}"
+ password: "{{misp_dbpass}}"
priv: "{{misp_dbname}}.*:ALL"
with_items: "{{groups['mispcontainers']}}"
-
-- name: Set Autostart for supervisord's services
- replace:
- path: /etc/supervisord.conf
- regexp: '^autostart=false$'
- replace: 'autostart=true'
+ tags:
+ - start
# CREATE DATABASE IF NOT EXISTS {{misp_dbname}};
# {% for misp_host in groups['mispcontainers'] %}
-# GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}}';
+# GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{misp_dbpass}}';
# GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}';
# {% endfor %}
#
diff --git a/roles/mysql/tasks/secure.yml b/roles/mysql/tasks/secure.yml
index 80db96cf207efd2ba5b4ec0c5aa4828041a6e980..18f098e0bf61973914ca8e52e4de00a7e8a10652 100644
--- a/roles/mysql/tasks/secure.yml
+++ b/roles/mysql/tasks/secure.yml
@@ -4,7 +4,7 @@
mysql_user:
name: root
host_all: yes
- password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}"
+ password: "{{mysql_dbrootpass}}"
tags:
- start
ignore_errors: true
@@ -54,7 +54,7 @@
#
#
#
-# UPDATE mysql.user SET Password=PASSWORD('{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}') WHERE User='root';
+# UPDATE mysql.user SET Password=PASSWORD('{{mysql_dbrootpass}}') WHERE User='root';
# DELETE FROM mysql.user WHERE User='';
# DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
# DROP DATABASE IF EXISTS test;
@@ -64,9 +64,9 @@
#
#
# #!/bin/bash -x
-# MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}})
-# #MISPINIT=$(echo "select count(id) from users;" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}})
+# MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}})
+# #MISPINIT=$(echo "select count(id) from users;" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}})
# if [ ${MISPINIT} == "0" ]; then
-# cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}}
+# cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}}
# touch /var/www/MISP/dbchecked-$(date +%Y%m%d_%H%M%S)
# fi
diff --git a/roles/mysql/templates/dotmy.cnf.j2 b/roles/mysql/templates/dotmy.cnf.j2
index 79fe59e355fa1a7d55872cdd80e0025bfc246f25..56feaea61621677a51089d18f6fea96fc3a30a4b 100644
--- a/roles/mysql/templates/dotmy.cnf.j2
+++ b/roles/mysql/templates/dotmy.cnf.j2
@@ -1,3 +1,3 @@
[client]
user=root
-password='{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}'
+password='{{mysql_dbrootpass}}'
diff --git a/roles/mysql/templates/mysql_secure.sql.j2 b/roles/mysql/templates/mysql_secure.sql.j2
index 5b8474e9b0dc5dbcc27f013a771f2187fdc68b6a..c1f602d390f8e986c6b8cf73664a2bde3a87fbbb 100644
--- a/roles/mysql/templates/mysql_secure.sql.j2
+++ b/roles/mysql/templates/mysql_secure.sql.j2
@@ -1,4 +1,4 @@
-UPDATE mysql.user SET Password=PASSWORD('{{playbook_dir}}/secrets/passwords/mysql_root')}}') WHERE User='root';
+UPDATE mysql.user SET Password=PASSWORD('{{mysql_dbrootpass}}') WHERE User='root';
DELETE FROM mysql.user WHERE User='';
DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
DROP DATABASE IF EXISTS test;
@@ -6,14 +6,9 @@ DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%';
CREATE DATABASE IF NOT EXISTS {{misp_dbname}};
{% for misp_host in groups['mispcontainers'] %}
-GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{playbook_dir}}/secrets/passwords/mysql_misp')}}';
+GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{misp_dbpass}}';
GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}';
{% endfor %}
-INSTALL SONAME 'server_audit';
-SET GLOBAL server_audit_logging=ON;
-SET GLOBAL server_audit_file_rotate_now = ON;
-SET GLOBAl server_audit_file_rotations = 2;
-SET GLOABL audit_log_format = JSON;
FLUSH PRIVILEGES;
diff --git a/roles/nifi/tasks/main.yml b/roles/nifi/tasks/main.yml
index d4ab3ad3a89413bf3d6a848822562a5ccb6d92ca..ec0249ccf785f4a8e9c55e3330d63787735ac4f0 100644
--- a/roles/nifi/tasks/main.yml
+++ b/roles/nifi/tasks/main.yml
@@ -1,17 +1,151 @@
---
-- include: start.yml
+- name: Copy cacert to ca-trust dir
+ remote_user: root
+ copy:
+ src: "files/{{ca_cn}}.crt"
+ dest: /etc/pki/ca-trust/source/anchors/ca.crt
+ tags:
+ - start
+
+- name: Install cacert to root truststore
+ remote_user: root
+ command: "update-ca-trust"
+ tags:
+ - start
+
+- name: Copy certificates in NiFi conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "conf/{{ item }}"
+ with_items:
+ - "{{ inventory_hostname }}.p12"
+ - cacerts.jks
+ tags:
+ - start
+
+- name: Configure flow.xml
+ template:
+ src: "flow.xml.j2"
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+ tags:
+ - start
+
+- name: Gzip flow.xml
+ archive:
+ path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+ format: gz
+ tags:
+ - start
+
+- name: Get openid authkey
+ set_fact:
+ nifisecret: "{{lookup('file', 'files/nifisecret',convert_data=False) | from_json }}"
+ tags:
+ - start
+
+- name: Configure NiFi boostrap properties
+ template:
+ src: bootstrap.conf.j2
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf"
+ tags:
+ - start
+
+- name: Configure NiFi properties for secure servers
+ template:
+ src: nifi.properties.j2
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties"
+ tags:
+ - start
+
+- name: Copy authorizations.xml
+ copy:
+ src: "authorizations.xml"
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml"
+ tags:
+ - start
+
+- name: Configure users
+ template:
+ src: users.xml.j2
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml"
+ tags:
+ - start
+
+- name: Configure NiFi authorizers for secure servers
+ template:
+ src: authorizers.xml.j2
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml"
+ tags:
+ - start
+
+- name: Create conf/enrich dir
+ file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory
+ tags:
+ - start
+
+- name: Copy empty GeoLite2-City database
+ copy:
+ src: GeoLite2-City.mmdb
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb"
+ tags:
+ - start
+
+- name: Download list of Tor exist nodes
+ get_url:
+ url: https://check.torproject.org/torbulkexitlist
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv"
+ tags:
+ - start
+
+- name: Add header to tornodes.csv
+ lineinfile:
+ path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv"
+ line: 'ip_addr'
+ insertbefore: BOF
tags:
- start
-- include: stop.yml
+
+- name: Download umbrella-top-1m.csv
+ get_url:
+ url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
tags:
- - stop
- - stop-nifi
-- include: update-config.yml
+ - start
+
+- name: Download alexa-top-1m.csv
+ get_url:
+ url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip
+ dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
+ tags:
+ - start
+
+- name: Start NiFi
+ command: "{{ ansible_facts.env['NIFI_HOME'] }}/bin/nifi.sh start"
tags:
- - update-config
- - update-nifi-config
-- include: restart.yml
+ - start
+
+#- name: check reachable hosts
+# gather_facts: no
+# tasks:
+# - command: ping -c1 {{ inventory_hostname }}
+# delegate_to: localhost
+# register: ping_result
+# ignore_errors: yes
+# - group_by: key=reachable
+# when: ping_result|success
+
+- name: Stop NiFi
+ command: "{{ ansible_facts.env['NIFI_HOME'] }}/bin/nifi.sh stop"
tags:
- - restart
- - restart-nifi
+ - stop
+
+#- name: Copy flow from NiFi
+# fetch:
+# src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+# dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
+# flat: yes
+# tags:
+# - stop
+
diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2
index ec452185ec9559dd620b084aaa89a74009cfc01e..20517182d06a1ba8cf6ab4063e8f1ff03b723d5b 100644
--- a/roles/nifi/templates/flow.xml.j2
+++ b/roles/nifi/templates/flow.xml.j2
@@ -8,22 +8,16 @@
<name>NiFi Flow</name>
<position x="0.0" y="0.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processGroup>
<id>41088add-955b-3611-a0de-2c18b79b678c</id>
<name>Data processing</name>
<position x="1216.0" y="256.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processGroup>
<id>fcbcacd1-542d-3a15-a5aa-9c1302328954</id>
<name>Enrichment</name>
<position x="384.0" y="720.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>01b66126-695a-3059-b179-f1bf85e8ca91</id>
<name>Check fqdn enrichment</name>
@@ -34,7 +28,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -65,7 +59,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -96,7 +90,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -136,8 +130,6 @@
<name>fqdn enrichment</name>
<position x="-202.47354083453774" y="596.1945491887745" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>2703fe3e-7e6a-310e-a010-b30898befa9b</id>
<name>Check fqdn2 enrichment</name>
@@ -148,7 +140,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -188,8 +180,6 @@
<name>Enrich fqdn2</name>
<position x="368.0" y="416.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>0cdf1224-a6d5-3ac4-a4f5-27a7588f1d98</id>
<name>Input</name>
@@ -200,7 +190,7 @@
<outputPort>
<id>58b0935e-1c5b-3ad1-a2cc-7de4826170a6</id>
<name>Output</name>
- <position x="1096.0" y="528.0" />
+ <position x="432.0" y="536.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -209,8 +199,6 @@
<name>Umbrella</name>
<position x="379.1725199638772" y="224.3012562119061" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>0306116f-b13d-30fc-94b1-34f3c8ba95da</id>
<name>Umbrella fqdn1</name>
@@ -221,7 +209,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -345,167 +333,6 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
- <processGroup>
- <id>f2263bf3-7929-182a-8971-2ac159d0cca9</id>
- <name>Misp lookup</name>
- <position x="1024.0" y="224.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>cadc3f2e-8e08-195c-898e-1b5fbcdf56e0</id>
- <name>Misp fqdn2</name>
- <position x="480.99999953064577" y="197.99999761468263" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.LookupRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>lookup-service</name>
- <value>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</value>
- </property>
- <property>
- <name>result-record-path</name>
- <value>${enrich_fqdn2}_misp</value>
- </property>
- <property>
- <name>routing-strategy</name>
- <value>route-to-success</value>
- </property>
- <property>
- <name>result-contents</name>
- <value>insert-entire-record</value>
- </property>
- <property>
- <name>record-update-strategy</name>
- <value>use-property</value>
- </property>
- <property>
- <name>key</name>
- <value>${enrich_fqdn2}</value>
- </property>
- </processor>
- <inputPort>
- <id>dca538c3-e563-1b63-8de2-edf46a599279</id>
- <name>Input</name>
- <position x="536.0" y="16.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>a4ef3d87-6241-14a5-b543-1824b197991c</id>
- <name>Output</name>
- <position x="536.0" y="456.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <funnel>
- <id>263b3963-3c73-1efc-8286-4e57645eefc2</id>
- <position x="152.0" y="248.0" />
- </funnel>
- <connection>
- <id>e47137a6-8c07-12bf-9d69-0e0c10b05088</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>cadc3f2e-8e08-195c-898e-1b5fbcdf56e0</sourceId>
- <sourceGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>a4ef3d87-6241-14a5-b543-1824b197991c</destinationId>
- <destinationGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>b4723c4d-5ed2-1f9e-bd7c-04076aa149cf</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>cadc3f2e-8e08-195c-898e-1b5fbcdf56e0</sourceId>
- <sourceGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>263b3963-3c73-1efc-8286-4e57645eefc2</destinationId>
- <destinationGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</destinationGroupId>
- <destinationType>FUNNEL</destinationType>
- <relationship>failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>60 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>edab3c6f-d21b-12a1-8b73-10c8012a3bd2</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>dca538c3-e563-1b63-8de2-edf46a599279</sourceId>
- <sourceGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>cadc3f2e-8e08-195c-898e-1b5fbcdf56e0</destinationId>
- <destinationGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <connection>
- <id>934921df-0175-1000-ffff-ffffeabcba7a</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>a4ef3d87-6241-14a5-b543-1824b197991c</sourceId>
- <sourceGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>58b0935e-1c5b-3ad1-a2cc-7de4826170a6</destinationId>
- <destinationGroupId>f9d9877a-de02-3374-9241-d3ca1939678b</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
<connection>
<id>fae5d513-1604-3fed-9adb-8ad19b5ab3c0</id>
<name />
@@ -515,9 +342,9 @@
<sourceId>c8994201-98f4-31d2-a604-11c3b454df00</sourceId>
<sourceGroupId>363fd695-1466-3404-ada3-036133ff8d15</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
- <destinationId>dca538c3-e563-1b63-8de2-edf46a599279</destinationId>
- <destinationGroupId>f2263bf3-7929-182a-8971-2ac159d0cca9</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
+ <destinationId>58b0935e-1c5b-3ad1-a2cc-7de4826170a6</destinationId>
+ <destinationGroupId>f9d9877a-de02-3374-9241-d3ca1939678b</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
@@ -552,8 +379,6 @@
<name>Enrich fqdn1</name>
<position x="376.0" y="200.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>f2cc04f8-56bc-3adb-8d72-6ad7f6e6e48c</id>
<name>Input</name>
@@ -564,7 +389,7 @@
<outputPort>
<id>12745a98-f547-38d2-9c50-a471e8cf6fc7</id>
<name>Output</name>
- <position x="1112.0" y="528.0" />
+ <position x="432.0" y="536.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -573,8 +398,6 @@
<name>Umbrella</name>
<position x="379.1725199638772" y="224.3012562119061" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>ce84ada1-58b8-3c28-bc5a-64fc4f39e008</id>
<name>Umbrella fqdn1</name>
@@ -585,7 +408,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -629,7 +452,6 @@
<name>key</name>
<value>${enrich_fqdn1}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>c31f9f4a-becc-35c3-b9c0-b836d061e364</id>
@@ -645,6 +467,10 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>8811d4ff-bf71-38a6-8cf0-e5732840e1de</id>
+ <position x="-1204.5503424650574" y="470.375701251353" />
+ </funnel>
<connection>
<id>99b11d95-dfb3-34cf-8657-0c141d1e9f63</id>
<name />
@@ -685,117 +511,19 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
- </processGroup>
- <processGroup>
- <id>1de23f54-e22a-19df-8dd2-9235ae248d29</id>
- <name>Misp lookup</name>
- <position x="1056.0" y="216.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>16d53f41-fc96-1292-b434-9a157e27eaf3</id>
- <name>Misp fqdn1</name>
- <position x="480.99999953064577" y="197.99999761468263" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.LookupRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>lookup-service</name>
- <value>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</value>
- </property>
- <property>
- <name>result-record-path</name>
- <value>${enrich_fqdn1}_misp</value>
- </property>
- <property>
- <name>routing-strategy</name>
- <value>route-to-success</value>
- </property>
- <property>
- <name>result-contents</name>
- <value>insert-entire-record</value>
- </property>
- <property>
- <name>record-update-strategy</name>
- <value>use-property</value>
- </property>
- <property>
- <name>key</name>
- <value>${enrich_fqdn1}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>844331e5-cd55-1c55-889b-d447dfba57a2</id>
- <name>Input</name>
- <position x="536.0" y="16.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>e13e32a5-f5d5-1f5e-a38e-c1e19453fd31</id>
- <name>Output</name>
- <position x="536.0" y="456.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
<connection>
- <id>6153303e-0c09-1d36-9b87-6c2d0a5ed15a</id>
+ <id>badc00ed-f022-3045-bcfd-8a4839fef4bf</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>16d53f41-fc96-1292-b434-9a157e27eaf3</sourceId>
- <sourceGroupId>1de23f54-e22a-19df-8dd2-9235ae248d29</sourceGroupId>
+ <sourceId>ce84ada1-58b8-3c28-bc5a-64fc4f39e008</sourceId>
+ <sourceGroupId>c763d1fd-ae0f-3297-bab7-abf22759310d</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>e13e32a5-f5d5-1f5e-a38e-c1e19453fd31</destinationId>
- <destinationGroupId>1de23f54-e22a-19df-8dd2-9235ae248d29</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>3a4133c4-bc18-1bb7-84a0-06b25aba73f4</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>844331e5-cd55-1c55-889b-d447dfba57a2</sourceId>
- <sourceGroupId>1de23f54-e22a-19df-8dd2-9235ae248d29</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>16d53f41-fc96-1292-b434-9a157e27eaf3</destinationId>
- <destinationGroupId>1de23f54-e22a-19df-8dd2-9235ae248d29</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
+ <destinationId>8811d4ff-bf71-38a6-8cf0-e5732840e1de</destinationId>
+ <destinationGroupId>c763d1fd-ae0f-3297-bab7-abf22759310d</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -833,26 +561,6 @@
<sourceId>9b6cb318-b9e8-36f1-bd8a-17b2aa38343a</sourceId>
<sourceGroupId>c763d1fd-ae0f-3297-bab7-abf22759310d</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
- <destinationId>844331e5-cd55-1c55-889b-d447dfba57a2</destinationId>
- <destinationGroupId>1de23f54-e22a-19df-8dd2-9235ae248d29</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>4f303cda-ab21-1acd-ffff-ffffcdd40e4b</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e13e32a5-f5d5-1f5e-a38e-c1e19453fd31</sourceId>
- <sourceGroupId>1de23f54-e22a-19df-8dd2-9235ae248d29</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
<destinationId>12745a98-f547-38d2-9c50-a471e8cf6fc7</destinationId>
<destinationGroupId>11b67527-3401-3961-97ba-425f721de3e0</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
@@ -971,8 +679,6 @@
<name>Custom enrichment</name>
<position x="-536.0" y="944.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>2ba9db29-0172-1000-ffff-ffffc1aa6db2</id>
<name>Input</name>
@@ -1013,8 +719,6 @@
<name>IP enrichment</name>
<position x="-216.0" y="16.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>be9226d7-777f-3820-bf10-49e97e9b73cc</id>
<name>Check IP2 enrichment</name>
@@ -1025,7 +729,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1065,8 +769,6 @@
<name>Enrich IP1</name>
<position x="-48.0" y="312.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>ff302e29-64bb-3e10-b76e-91a13d2470d8</id>
<name>Input</name>
@@ -1086,8 +788,6 @@
<name>Misp lookup</name>
<position x="1160.0" y="136.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>68dc57d7-2016-3e27-ac1f-092c63909a63</id>
<name>Misp ip1</name>
@@ -1098,7 +798,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>16</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1142,7 +842,6 @@
<name>key</name>
<value>${enrich_ip1}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>57c1773b-65c7-3cdb-b161-a44fa977291a</id>
@@ -1158,6 +857,10 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>3f92a75f-ec3e-3f4c-9467-e5c0e981f032</id>
+ <position x="-1312.0" y="792.0" />
+ </funnel>
<connection>
<id>7f5a13cb-a049-385d-9ebc-3ef4aa8ff541</id>
<name />
@@ -1198,14 +901,32 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>d6c427a3-f4fb-39a5-b399-ebd9b0482039</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>68dc57d7-2016-3e27-ac1f-092c63909a63</sourceId>
+ <sourceGroupId>9b3b9cc7-49bf-3b66-b213-09f2f5df634e</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>3f92a75f-ec3e-3f4c-9467-e5c0e981f032</destinationId>
+ <destinationGroupId>9b3b9cc7-49bf-3b66-b213-09f2f5df634e</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
</processGroup>
<processGroup>
<id>88a2dcc1-ccce-3396-9f4d-bc1e31c70063</id>
<name>GeoIP</name>
<position x="480.0" y="144.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>fb1a97b2-5e15-3bca-9284-f23a9cc90eed</id>
<name>Add GeoIP fields</name>
@@ -1216,7 +937,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1264,19 +985,18 @@
<name>pretty_print</name>
<value>false</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
<id>11668896-2c3e-3712-905a-eb406bf33c2a</id>
<name>GeoIP IP</name>
- <position x="-568.0" y="160.0" />
+ <position x="-1192.0" y="424.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.GeoEnrichIPRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-enrich-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>12</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1337,62 +1057,6 @@
</property>
<autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
- <processor>
- <id>349b3525-a821-1197-0000-00006e02758e</id>
- <name>Contry code to region</name>
- <position x="-1216.0" y="368.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.LookupRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>lookup-service</name>
- <value>349b34c7-a821-1197-ffff-ffff85d82877</value>
- </property>
- <property>
- <name>result-record-path</name>
- <value>${enrich_ip1:substring(1):contains('/'):ifElse('${enrich_ip1:substringBeforeLast(\"/\")}',${enrich_ip1:append('_')}):append("/geo/region")}</value>
- </property>
- <property>
- <name>routing-strategy</name>
- <value>route-to-success</value>
- </property>
- <property>
- <name>result-contents</name>
- <value>insert-entire-record</value>
- </property>
- <property>
- <name>record-update-strategy</name>
- <value>use-property</value>
- </property>
- <property>
- <name>key</name>
- <value>${enrich_ip1:substring(1):contains('/'):ifElse('${enrich_ip1:substringBeforeLast(\"/\")}',${enrich_ip1:append('_')}):append("/geo/country_iso_code")}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
<inputPort>
<id>e6d573f8-a297-3611-a1ff-f1918a2cbabc</id>
<name>Data Input</name>
@@ -1403,10 +1067,14 @@
<outputPort>
<id>17c34a1f-8048-366b-8ef4-107ce16a100e</id>
<name>Data output</name>
- <position x="-1168.0" y="656.0" />
+ <position x="-1160.0" y="752.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>b68c4522-cddb-30bf-8545-73a2644cd35e</id>
+ <position x="-1560.0" y="336.0" />
+ </funnel>
<connection>
<id>790863c1-3cfc-383f-ab9f-5d7ead5c0fe1</id>
<name />
@@ -1456,9 +1124,9 @@
<sourceId>11668896-2c3e-3712-905a-eb406bf33c2a</sourceId>
<sourceGroupId>88a2dcc1-ccce-3396-9f4d-bc1e31c70063</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>349b3525-a821-1197-0000-00006e02758e</destinationId>
+ <destinationId>17c34a1f-8048-366b-8ef4-107ce16a100e</destinationId>
<destinationGroupId>88a2dcc1-ccce-3396-9f4d-bc1e31c70063</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship>found</relationship>
<relationship>not found</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
@@ -1469,18 +1137,18 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>349b3542-a821-1197-0000-000059777de4</id>
+ <id>c831c34b-2a1a-3d24-963b-079b00961407</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>349b3525-a821-1197-0000-00006e02758e</sourceId>
+ <sourceId>fb1a97b2-5e15-3bca-9284-f23a9cc90eed</sourceId>
<sourceGroupId>88a2dcc1-ccce-3396-9f4d-bc1e31c70063</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>17c34a1f-8048-366b-8ef4-107ce16a100e</destinationId>
+ <destinationId>b68c4522-cddb-30bf-8545-73a2644cd35e</destinationId>
<destinationGroupId>88a2dcc1-ccce-3396-9f4d-bc1e31c70063</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -1494,8 +1162,6 @@
<name>Tor nodes lookup</name>
<position x="480.0" y="384.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>fd52d735-a256-3c52-9b90-fbe71f010fbe</id>
<name>Tor src IP</name>
@@ -1506,7 +1172,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>12</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1528,7 +1194,7 @@
</property>
<property>
<name>lookup-service</name>
- <value>bbd4d3a2-0175-1000-0000-00000b0fb8bd</value>
+ <value>bf81debc-0171-1000-0000-00002936ae5a</value>
</property>
<property>
<name>result-record-path</name>
@@ -1550,7 +1216,6 @@
<name>key</name>
<value>${enrich_ip1}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>eb136a03-4809-3d2f-a85e-e739ffa665cd</id>
@@ -1566,6 +1231,10 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>44794379-111e-3f28-bf84-bf5850f42587</id>
+ <position x="-1412.2104124778762" y="829.0775745939713" />
+ </funnel>
<connection>
<id>c681313a-f2c8-3439-8c00-70f9783fea67</id>
<name />
@@ -1606,6 +1275,26 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>ad32e629-d5a2-38b7-a50e-9acf0ad9d768</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>fd52d735-a256-3c52-9b90-fbe71f010fbe</sourceId>
+ <sourceGroupId>3cb64c25-8af7-361d-b6e7-e002defe7411</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>44794379-111e-3f28-bf84-bf5850f42587</destinationId>
+ <destinationGroupId>3cb64c25-8af7-361d-b6e7-e002defe7411</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
</processGroup>
<connection>
<id>3831c5df-c2a8-3a1b-9d67-8f37abf05365</id>
@@ -1693,8 +1382,6 @@
<name>Enrich IP2</name>
<position x="-56.0" y="584.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>742a8211-a5e6-347d-9e79-25facc6b181e</id>
<name>Input</name>
@@ -1714,75 +1401,17 @@
<name>GeoIP and IPreg</name>
<position x="480.0" y="144.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>9ed935e9-7bd4-150a-913d-e6a4dedb2939</id>
- <name>Contry code to region</name>
- <position x="-1224.0" y="368.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.LookupRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>lookup-service</name>
- <value>349b34c7-a821-1197-ffff-ffff85d82877</value>
- </property>
- <property>
- <name>result-record-path</name>
- <value>${enrich_ip2:substring(1):contains('/'):ifElse('${enrich_ip2:substringBeforeLast(\"/\")}',${enrich_ip2:append('_')}):append("/geo/region")}</value>
- </property>
- <property>
- <name>routing-strategy</name>
- <value>route-to-success</value>
- </property>
- <property>
- <name>result-contents</name>
- <value>insert-entire-record</value>
- </property>
- <property>
- <name>record-update-strategy</name>
- <value>use-property</value>
- </property>
- <property>
- <name>key</name>
- <value>${enrich_ip2:substring(1):contains('/'):ifElse('${enrich_ip2:substringBeforeLast(\"/\")}',${enrich_ip2:append('_')}):append("/geo/country_iso_code")}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
<processor>
<id>e16581ea-ff17-3cec-bd39-b1fe52797bd2</id>
<name>GeoIP IP</name>
- <position x="-592.0" y="168.0" />
+ <position x="-1232.0" y="432.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.GeoEnrichIPRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-enrich-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>16</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1853,7 +1482,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -1901,7 +1530,6 @@
<name>pretty_print</name>
<value>false</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>aa73b938-03ee-3d90-ba6f-0998b568ac36</id>
@@ -1917,19 +1545,23 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>8e8abe4a-9ed6-3ec8-9e9d-719242be1a2b</id>
+ <position x="-1560.0" y="336.0" />
+ </funnel>
<connection>
- <id>9ed935f0-7bd4-150a-ffff-ffffaed76b62</id>
+ <id>a4870f06-f96b-328a-83d8-4eeb6db5cc93</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>9ed935e9-7bd4-150a-913d-e6a4dedb2939</sourceId>
+ <sourceId>d467bca6-15df-38bb-8f9b-65ae1f9dceaa</sourceId>
<sourceGroupId>a8873c56-d149-34d1-8344-a35b339e6187</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>0dbb2fcb-c11b-396e-8918-24d7121f7653</destinationId>
+ <destinationId>8e8abe4a-9ed6-3ec8-9e9d-719242be1a2b</destinationId>
<destinationGroupId>a8873c56-d149-34d1-8344-a35b339e6187</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -1966,9 +1598,9 @@
<sourceId>e16581ea-ff17-3cec-bd39-b1fe52797bd2</sourceId>
<sourceGroupId>a8873c56-d149-34d1-8344-a35b339e6187</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>9ed935e9-7bd4-150a-913d-e6a4dedb2939</destinationId>
+ <destinationId>0dbb2fcb-c11b-396e-8918-24d7121f7653</destinationId>
<destinationGroupId>a8873c56-d149-34d1-8344-a35b339e6187</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship>found</relationship>
<relationship>not found</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
@@ -2004,8 +1636,6 @@
<name>Tor nodes lookup</name>
<position x="480.0" y="384.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>adae2d07-ad4f-38f2-9a8c-b7638863cac7</id>
<name>Tor src IP</name>
@@ -2016,7 +1646,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>16</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -2038,7 +1668,7 @@
</property>
<property>
<name>lookup-service</name>
- <value>bbd4d3a2-0175-1000-0000-00000b0fb8bd</value>
+ <value>bf81debc-0171-1000-0000-00002936ae5a</value>
</property>
<property>
<name>result-record-path</name>
@@ -2060,7 +1690,6 @@
<name>key</name>
<value>${enrich_ip2}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>1c42374a-61ad-3d92-9f86-1f1ae9bae6d0</id>
@@ -2076,6 +1705,10 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>4bad43e2-bfed-3e45-a9d1-fdabb2ab85ea</id>
+ <position x="-1403.592919640183" y="819.7224736847786" />
+ </funnel>
<connection>
<id>0b2f9272-06fe-3a8c-a322-9972ac5c0466</id>
<name />
@@ -2116,14 +1749,34 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>fe67c36a-4ae0-3e30-8aad-a988a481df4b</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-1088.0" y="832.0" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>adae2d07-ad4f-38f2-9a8c-b7638863cac7</sourceId>
+ <sourceGroupId>33258978-2057-3612-be77-0a763431ee29</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>4bad43e2-bfed-3e45-a9d1-fdabb2ab85ea</destinationId>
+ <destinationGroupId>33258978-2057-3612-be77-0a763431ee29</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
</processGroup>
<processGroup>
<id>de2bc05d-fbd2-35bc-9192-b82041176492</id>
<name>Misp lookup</name>
<position x="1160.0" y="136.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>2cd2029e-53ae-3575-bf35-785203683c7f</id>
<name>Misp ip</name>
@@ -2134,7 +1787,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>12</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -2178,7 +1831,6 @@
<name>key</name>
<value>${enrich_ip2}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>d6490898-79c0-3eb5-a102-718ff8ab68e6</id>
@@ -2194,6 +1846,10 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>d259fdb6-aa03-35b1-b439-b5baccc57a1e</id>
+ <position x="-1264.0" y="784.0" />
+ </funnel>
<connection>
<id>d2bfc9ce-5c85-3dc2-a0c5-83bbf0c77277</id>
<name />
@@ -2215,7 +1871,7 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>2a95cb88-0177-1000-ffff-ffffc8a50166</id>
+ <id>2475c41e-5711-3de0-bbec-2e977cd1f40a</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
@@ -2234,6 +1890,26 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>a7505eac-df84-3e17-8c40-687c159877a7</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>2cd2029e-53ae-3575-bf35-785203683c7f</sourceId>
+ <sourceGroupId>de2bc05d-fbd2-35bc-9192-b82041176492</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>d259fdb6-aa03-35b1-b439-b5baccc57a1e</destinationId>
+ <destinationGroupId>de2bc05d-fbd2-35bc-9192-b82041176492</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
</processGroup>
<connection>
<id>2ba90add-0172-1000-ffff-ffffb69e0af7</id>
@@ -2422,8 +2098,6 @@
<name>Domain enrichment</name>
<position x="-213.29739929099082" y="283.59665734060223" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>1957f5c3-b6cb-3c80-b527-1858c61ed111</id>
<name>Check domain2 enrichment</name>
@@ -2434,7 +2108,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -2474,8 +2148,6 @@
<name>Enrich domain1</name>
<position x="448.0" y="248.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>bf74c1ba-45ad-3c0d-84c6-ec7ea7bdcdea</id>
<name>Input</name>
@@ -2486,7 +2158,7 @@
<outputPort>
<id>f270695c-edf6-3a9f-bc3a-10804f26f56f</id>
<name>Output</name>
- <position x="528.0" y="864.0" />
+ <position x="1232.0" y="600.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -2495,8 +2167,6 @@
<name>Entropy</name>
<position x="1152.0" y="288.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>bf907759-0171-1000-0000-0000022727cd</id>
<name>Calculate entropy on domain1</name>
@@ -2507,15 +2177,15 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-scripting-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>ERROR</bulletinLevel>
+ <bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
@@ -2525,18 +2195,18 @@
</property>
<property>
<name>Script File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq/freqProcessor.py</value>
+ <value>/opt/nifi/nifi-current/conf/freq/freqProcessor.py</value>
</property>
<property>
<name>Script Body</name>
</property>
<property>
<name>Module Directory</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq</value>
+ <value>/opt/nifi/nifi-current/conf/freq</value>
</property>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>Frequency File</name>
+ <value>/opt/nifi/nifi-current/conf/freq/alexa.json</value>
</property>
<property>
<name>record-writer</name>
@@ -2547,18 +2217,17 @@
<value>${enrich_domain1}</value>
</property>
<property>
- <name>Result Field 1</name>
- <value>${enrich_domain1}_freq1</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>Result Field 2</name>
<value>${enrich_domain1}_freq2</value>
</property>
<property>
- <name>Frequency File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value>
+ <name>Result Field 1</name>
+ <value>${enrich_domain1}_freq1</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>84dfe3a1-a6f4-3b80-9161-93a59b88ad8b</id>
@@ -2574,6 +2243,10 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>7297c375-d15a-36a2-a115-6b2680fbef2b</id>
+ <position x="128.0" y="360.0" />
+ </funnel>
<connection>
<id>2bea3770-0172-1000-ffff-ffff9cd568b2</id>
<name />
@@ -2594,18 +2267,38 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>bf938168-0171-1000-0000-00003e6fd9c5</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>bf907759-0171-1000-0000-0000022727cd</sourceId>
+ <sourceGroupId>89e761fa-669e-30c2-9590-ab8f806ffaff</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>7297c375-d15a-36a2-a115-6b2680fbef2b</destinationId>
+ <destinationGroupId>89e761fa-669e-30c2-9590-ab8f806ffaff</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
<connection>
<id>27f7d6f4-0172-1000-ffff-ffffc65ffd5e</id>
<name />
<bendPoints />
- <labelIndex>0</labelIndex>
+ <labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>84dfe3a1-a6f4-3b80-9161-93a59b88ad8b</sourceId>
<sourceGroupId>89e761fa-669e-30c2-9590-ab8f806ffaff</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
- <destinationId>bf907759-0171-1000-0000-0000022727cd</destinationId>
+ <destinationId>2ef455b6-6c77-39ae-87bf-d21b988f4c52</destinationId>
<destinationGroupId>89e761fa-669e-30c2-9590-ab8f806ffaff</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
@@ -2616,25 +2309,23 @@
</connection>
</processGroup>
<processGroup>
- <id>9ed9363a-7bd4-150a-0000-00000cc58ac4</id>
- <name>Misp lookup</name>
- <position x="456.0" y="552.0" />
+ <id>df54305f-551e-38ff-8e1c-a1c272c238c0</id>
+ <name>Alexa</name>
+ <position x="448.0" y="288.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
- <id>9ed93644-7bd4-150a-0000-00003c8c3388</id>
- <name>Misp domain1</name>
- <position x="480.99999953064577" y="197.99999761468263" />
+ <id>ccab62e9-53cd-3d5e-aca3-6fa2e1b95597</id>
+ <name>Alexa domain1</name>
+ <position x="-726.6263417228297" y="457.36850568010186" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.LookupRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
+ <maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
@@ -2643,7 +2334,7 @@
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
+ <runDurationNanos>25000000</runDurationNanos>
<property>
<name>record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
@@ -2654,11 +2345,11 @@
</property>
<property>
<name>lookup-service</name>
- <value>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</value>
+ <value>282db77b-3118-3b3c-bbd1-e4260b1c6395</value>
</property>
<property>
<name>result-record-path</name>
- <value>${enrich_domain1}_misp</value>
+ <value>${enrich_domain1}_alexa</value>
</property>
<property>
<name>routing-strategy</name>
@@ -2676,33 +2367,36 @@
<name>key</name>
<value>${enrich_domain1}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
- <id>9ed93645-7bd4-150a-ffff-fffff6fa372f</id>
- <name>Input</name>
- <position x="536.0" y="16.0" />
+ <id>c7b3b6c4-6418-3dbe-a037-387e3c85cb19</id>
+ <name>Input data</name>
+ <position x="-673.6437977775172" y="241.37341901017993" />
<comments />
<scheduledState>RUNNING</scheduledState>
</inputPort>
<outputPort>
- <id>9ed9364e-7bd4-150a-ffff-ffffa5012b2a</id>
- <name>Output</name>
- <position x="536.0" y="456.0" />
+ <id>ff4ada2c-5f6f-33bc-ae0c-8cdb24dd2357</id>
+ <name>Output data</name>
+ <position x="-704.0" y="736.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>85764070-cd41-38a1-a352-929737e563c6</id>
+ <position x="-1152.0" y="512.0" />
+ </funnel>
<connection>
- <id>9ed9364f-7bd4-150a-0000-000019b6beab</id>
+ <id>002d4937-dcb5-3cf1-8075-8d5533667510</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>9ed93644-7bd4-150a-0000-00003c8c3388</sourceId>
- <sourceGroupId>9ed9363a-7bd4-150a-0000-00000cc58ac4</sourceGroupId>
+ <sourceId>ccab62e9-53cd-3d5e-aca3-6fa2e1b95597</sourceId>
+ <sourceGroupId>df54305f-551e-38ff-8e1c-a1c272c238c0</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>9ed9364e-7bd4-150a-ffff-ffffa5012b2a</destinationId>
- <destinationGroupId>9ed9363a-7bd4-150a-0000-00000cc58ac4</destinationGroupId>
+ <destinationId>ff4ada2c-5f6f-33bc-ae0c-8cdb24dd2357</destinationId>
+ <destinationGroupId>df54305f-551e-38ff-8e1c-a1c272c238c0</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
@@ -2713,116 +2407,18 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>9ed93647-7bd4-150a-0000-00000e5782ec</id>
+ <id>1af92349-7a45-30ec-b6fc-6d881dcd8b26</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>9ed93645-7bd4-150a-ffff-fffff6fa372f</sourceId>
- <sourceGroupId>9ed9363a-7bd4-150a-0000-00000cc58ac4</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>9ed93644-7bd4-150a-0000-00003c8c3388</destinationId>
- <destinationGroupId>9ed9363a-7bd4-150a-0000-00000cc58ac4</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>df54305f-551e-38ff-8e1c-a1c272c238c0</id>
- <name>Alexa</name>
- <position x="448.0" y="288.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>ccab62e9-53cd-3d5e-aca3-6fa2e1b95597</id>
- <name>Alexa domain1</name>
- <position x="-726.6263417228297" y="457.36850568010186" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.LookupRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>8</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>25000000</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>lookup-service</name>
- <value>282db77b-3118-3b3c-bbd1-e4260b1c6395</value>
- </property>
- <property>
- <name>result-record-path</name>
- <value>${enrich_domain1}_alexa</value>
- </property>
- <property>
- <name>routing-strategy</name>
- <value>route-to-success</value>
- </property>
- <property>
- <name>result-contents</name>
- <value>insert-entire-record</value>
- </property>
- <property>
- <name>record-update-strategy</name>
- <value>use-property</value>
- </property>
- <property>
- <name>key</name>
- <value>${enrich_domain1}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>c7b3b6c4-6418-3dbe-a037-387e3c85cb19</id>
- <name>Input data</name>
- <position x="-673.6437977775172" y="241.37341901017993" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>ff4ada2c-5f6f-33bc-ae0c-8cdb24dd2357</id>
- <name>Output data</name>
- <position x="-704.0" y="736.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>002d4937-dcb5-3cf1-8075-8d5533667510</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>ccab62e9-53cd-3d5e-aca3-6fa2e1b95597</sourceId>
- <sourceGroupId>df54305f-551e-38ff-8e1c-a1c272c238c0</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>ff4ada2c-5f6f-33bc-ae0c-8cdb24dd2357</destinationId>
- <destinationGroupId>df54305f-551e-38ff-8e1c-a1c272c238c0</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
+ <sourceId>ccab62e9-53cd-3d5e-aca3-6fa2e1b95597</sourceId>
+ <sourceGroupId>df54305f-551e-38ff-8e1c-a1c272c238c0</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>85764070-cd41-38a1-a352-929737e563c6</destinationId>
+ <destinationGroupId>df54305f-551e-38ff-8e1c-a1c272c238c0</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -2851,26 +2447,6 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
</processGroup>
- <connection>
- <id>9ed93675-7bd4-150a-0000-000017d6cf1e</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>9ed9364e-7bd4-150a-ffff-ffffa5012b2a</sourceId>
- <sourceGroupId>9ed9363a-7bd4-150a-0000-00000cc58ac4</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>f270695c-edf6-3a9f-bc3a-10804f26f56f</destinationId>
- <destinationGroupId>309911c5-0f97-37e4-8511-3c5639f5db10</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
<connection>
<id>ff7df1ed-63be-3637-aa15-094355c9711f</id>
<name />
@@ -2900,9 +2476,9 @@
<sourceId>2ef455b6-6c77-39ae-87bf-d21b988f4c52</sourceId>
<sourceGroupId>89e761fa-669e-30c2-9590-ab8f806ffaff</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
- <destinationId>9ed93645-7bd4-150a-ffff-fffff6fa372f</destinationId>
- <destinationGroupId>9ed9363a-7bd4-150a-0000-00000cc58ac4</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
+ <destinationId>f270695c-edf6-3a9f-bc3a-10804f26f56f</destinationId>
+ <destinationGroupId>309911c5-0f97-37e4-8511-3c5639f5db10</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
@@ -2937,8 +2513,6 @@
<name>Enrich domain2</name>
<position x="440.0" y="496.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<inputPort>
<id>0097ae66-e4c5-316c-b301-c8fd4481cbf4</id>
<name>Input</name>
@@ -2949,135 +2523,15 @@
<outputPort>
<id>1464f185-b2d5-3347-8aaa-89cfc91566f5</id>
<name>Output</name>
- <position x="528.0" y="816.0" />
+ <position x="1232.0" y="600.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <processGroup>
- <id>3514302f-5e74-1185-b5c5-edc034d911a6</id>
- <name>Misp lookup</name>
- <position x="448.0" y="520.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>2a1b3c82-c06f-1184-b6ec-245bb0032b5a</id>
- <name>Misp domain1</name>
- <position x="480.99999953064577" y="197.99999761468263" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.LookupRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>lookup-service</name>
- <value>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</value>
- </property>
- <property>
- <name>result-record-path</name>
- <value>${enrich_domain2}_misp</value>
- </property>
- <property>
- <name>routing-strategy</name>
- <value>route-to-success</value>
- </property>
- <property>
- <name>result-contents</name>
- <value>insert-entire-record</value>
- </property>
- <property>
- <name>record-update-strategy</name>
- <value>use-property</value>
- </property>
- <property>
- <name>key</name>
- <value>${enrich_domain2}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>7f5c3719-3707-1577-ad0d-945c5da3a77b</id>
- <name>Input</name>
- <position x="536.0" y="16.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>af083d3d-f063-1506-a7fe-2c6f0864de22</id>
- <name>Output</name>
- <position x="536.0" y="456.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>9cf430ed-a82c-1272-853f-4eeb68c144e2</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>2a1b3c82-c06f-1184-b6ec-245bb0032b5a</sourceId>
- <sourceGroupId>3514302f-5e74-1185-b5c5-edc034d911a6</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>af083d3d-f063-1506-a7fe-2c6f0864de22</destinationId>
- <destinationGroupId>3514302f-5e74-1185-b5c5-edc034d911a6</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>96023b7a-7902-1e45-91b0-0396e8bf647b</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>7f5c3719-3707-1577-ad0d-945c5da3a77b</sourceId>
- <sourceGroupId>3514302f-5e74-1185-b5c5-edc034d911a6</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>2a1b3c82-c06f-1184-b6ec-245bb0032b5a</destinationId>
- <destinationGroupId>3514302f-5e74-1185-b5c5-edc034d911a6</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
<processGroup>
<id>db261d6a-ab4a-3a2e-854a-cd42492788c5</id>
<name>Entropy</name>
<position x="1152.0" y="288.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>eff73930-f310-1a2e-b248-812b172b7415</id>
<name>Calculate entropy on domain2</name>
@@ -3088,13 +2542,13 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-scripting-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>ERROR</bulletinLevel>
+ <bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
@@ -3106,18 +2560,18 @@
</property>
<property>
<name>Script File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq/freqProcessor.py</value>
+ <value>/opt/nifi/nifi-current/conf/freq/freqProcessor.py</value>
</property>
<property>
<name>Script Body</name>
</property>
<property>
<name>Module Directory</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq</value>
+ <value>/opt/nifi/nifi-current/conf/freq</value>
</property>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>Frequency File</name>
+ <value>/opt/nifi/nifi-current/conf/freq/alexa.json</value>
</property>
<property>
<name>record-writer</name>
@@ -3128,18 +2582,17 @@
<value>${enrich_domain2}</value>
</property>
<property>
- <name>Result Field 1</name>
- <value>${enrich_domain2}_freq1</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>Result Field 2</name>
<value>${enrich_domain2}_freq2</value>
</property>
<property>
- <name>Frequency File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/freq/alexa.json</value>
+ <name>Result Field 1</name>
+ <value>${enrich_domain2}_freq1</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>ecc214d7-667d-3d81-affa-93e712a87abe</id>
@@ -3155,18 +2608,22 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>1667ea62-481a-3dfb-8745-8033f80ab5a7</id>
+ <position x="144.0" y="352.0" />
+ </funnel>
<connection>
<id>bf99e121-0171-1000-0000-0000487c681e</id>
<name />
<bendPoints />
- <labelIndex>0</labelIndex>
+ <labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>ecc214d7-667d-3d81-affa-93e712a87abe</sourceId>
<sourceGroupId>db261d6a-ab4a-3a2e-854a-cd42492788c5</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
- <destinationId>eff73930-f310-1a2e-b248-812b172b7415</destinationId>
+ <destinationId>10c439ea-f35c-3171-836c-d83af1403698</destinationId>
<destinationGroupId>db261d6a-ab4a-3a2e-854a-cd42492788c5</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
@@ -3195,14 +2652,32 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
+ <connection>
+ <id>bf9a2f76-0171-1000-ffff-fffffcafce73</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>eff73930-f310-1a2e-b248-812b172b7415</sourceId>
+ <sourceGroupId>db261d6a-ab4a-3a2e-854a-cd42492788c5</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>1667ea62-481a-3dfb-8745-8033f80ab5a7</destinationId>
+ <destinationGroupId>db261d6a-ab4a-3a2e-854a-cd42492788c5</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
</processGroup>
<processGroup>
<id>bd4c02d9-77f7-3984-8aa0-ad78099693a7</id>
<name>Alexa</name>
<position x="448.0" y="288.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>4598e8b8-b54a-3855-b6b7-5ac2721745a2</id>
<name>Alexa domain1</name>
@@ -3213,7 +2688,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>8</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -3257,7 +2732,6 @@
<name>key</name>
<value>${enrich_domain2}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<inputPort>
<id>a7e674e2-60b2-3417-ad31-d69248774c3f</id>
@@ -3273,19 +2747,23 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
+ <funnel>
+ <id>bf17534a-3723-3f7d-9ca1-dc1b61022e90</id>
+ <position x="-1152.0" y="512.0" />
+ </funnel>
<connection>
- <id>6580ec44-ce8d-3aaf-b7ba-ee4c4d67321b</id>
+ <id>3264d31c-6569-3dde-b174-ddff9f7d3835</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>a7e674e2-60b2-3417-ad31-d69248774c3f</sourceId>
+ <sourceId>4598e8b8-b54a-3855-b6b7-5ac2721745a2</sourceId>
<sourceGroupId>bd4c02d9-77f7-3984-8aa0-ad78099693a7</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>4598e8b8-b54a-3855-b6b7-5ac2721745a2</destinationId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>bf17534a-3723-3f7d-9ca1-dc1b61022e90</destinationId>
<destinationGroupId>bd4c02d9-77f7-3984-8aa0-ad78099693a7</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -3294,12 +2772,32 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>bd6b2b7a-04db-3e73-8f38-4520b7b07965</id>
+ <id>6580ec44-ce8d-3aaf-b7ba-ee4c4d67321b</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>4598e8b8-b54a-3855-b6b7-5ac2721745a2</sourceId>
+ <sourceId>a7e674e2-60b2-3417-ad31-d69248774c3f</sourceId>
+ <sourceGroupId>bd4c02d9-77f7-3984-8aa0-ad78099693a7</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>4598e8b8-b54a-3855-b6b7-5ac2721745a2</destinationId>
+ <destinationGroupId>bd4c02d9-77f7-3984-8aa0-ad78099693a7</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>bd6b2b7a-04db-3e73-8f38-4520b7b07965</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>4598e8b8-b54a-3855-b6b7-5ac2721745a2</sourceId>
<sourceGroupId>bd4c02d9-77f7-3984-8aa0-ad78099693a7</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>23f74eb2-f2ec-3612-b370-d18952393be5</destinationId>
@@ -3354,26 +2852,6 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
- <connection>
- <id>96023b92-7902-1e45-0000-000012465abe</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>af083d3d-f063-1506-a7fe-2c6f0864de22</sourceId>
- <sourceGroupId>3514302f-5e74-1185-b5c5-edc034d911a6</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>1464f185-b2d5-3347-8aaa-89cfc91566f5</destinationId>
- <destinationGroupId>464af303-e93a-32ed-a7ef-d3d553054447</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
<connection>
<id>7fcacebd-9235-37f8-9160-496d27d96a2d</id>
<name />
@@ -3383,9 +2861,9 @@
<sourceId>10c439ea-f35c-3171-836c-d83af1403698</sourceId>
<sourceGroupId>db261d6a-ab4a-3a2e-854a-cd42492788c5</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
- <destinationId>7f5c3719-3707-1577-ad0d-945c5da3a77b</destinationId>
- <destinationGroupId>3514302f-5e74-1185-b5c5-edc034d911a6</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
+ <destinationId>1464f185-b2d5-3347-8aaa-89cfc91566f5</destinationId>
+ <destinationGroupId>464af303-e93a-32ed-a7ef-d3d553054447</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
@@ -3724,16 +3202,16 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
<name>csv-file</name>
- <value>/opt/nifi/nifi-current/conf/enrich/alexa-top-1m.csv</value>
+ <value>/opt/nifi/nifi-current/conf/alexa-top-1m.csv</value>
</property>
<property>
<name>CSV Format</name>
- <value>default</value>
+ <value>Default</value>
</property>
<property>
<name>Character Set</name>
@@ -3744,36 +3222,13 @@
<value>domain</value>
</property>
<property>
- <name>ignore-duplicates</name>
- <value>true</value>
- </property>
- <property>
- <name>Value Separator</name>
- <value>,</value>
- </property>
- <property>
- <name>Quote Character</name>
- <value>"</value>
- </property>
- <property>
- <name>Quote Mode</name>
- <value>MINIMAL</value>
- </property>
- <property>
- <name>Comment Marker</name>
- </property>
- <property>
- <name>Escape Character</name>
- <value>\</value>
+ <name>lookup-value-column</name>
+ <value>index</value>
</property>
<property>
- <name>Trim Fields</name>
+ <name>ignore-duplicates</name>
<value>true</value>
</property>
- <property>
- <name>lookup-value-column</name>
- <value>index</value>
- </property>
</controllerService>
<controllerService>
<id>67289e27-a14d-3fa6-bcf9-91f7d2ae8d59</id>
@@ -3783,16 +3238,16 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
<name>csv-file</name>
- <value>/opt/nifi/nifi-current/conf/enrich/umbrella-top-1m.csv</value>
+ <value>/opt/nifi/nifi-current/conf/umbrella-top-1m.csv</value>
</property>
<property>
<name>CSV Format</name>
- <value>default</value>
+ <value>Excel</value>
</property>
<property>
<name>Character Set</name>
@@ -3803,36 +3258,13 @@
<value>domain</value>
</property>
<property>
- <name>ignore-duplicates</name>
- <value>true</value>
- </property>
- <property>
- <name>Value Separator</name>
- <value>,</value>
- </property>
- <property>
- <name>Quote Character</name>
- <value>"</value>
- </property>
- <property>
- <name>Quote Mode</name>
- <value>MINIMAL</value>
- </property>
- <property>
- <name>Comment Marker</name>
- </property>
- <property>
- <name>Escape Character</name>
- <value>\</value>
+ <name>lookup-value-column</name>
+ <value>index</value>
</property>
<property>
- <name>Trim Fields</name>
+ <name>ignore-duplicates</name>
<value>true</value>
</property>
- <property>
- <name>lookup-value-column</name>
- <value>index</value>
- </property>
</controllerService>
</processGroup>
<processGroup>
@@ -3840,8 +3272,6 @@
<name>Data input</name>
<position x="830.4597621124223" y="407.3463126314215" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
<id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
<name>To data output</name>
@@ -3852,7 +3282,7 @@
<outputPort>
<id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
<name>To enrichment</name>
- <position x="480.0" y="392.0" />
+ <position x="480.0" y="504.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
@@ -3861,8 +3291,6 @@
<name>Custom data inputs</name>
<position x="-504.0" y="952.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
<id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
<name>To data output</name>
@@ -3879,23 +3307,21 @@
</outputPort>
</processGroup>
<processGroup>
- <id>0c790562-0175-1000-ffff-ffffeaaeafc3</id>
- <name>FileBeat</name>
- <position x="-496.0" y="344.0" />
+ <id>84607b52-9748-3d38-b519-b0a05cddd097</id>
+ <name>Nifi logs</name>
+ <position x="-496.0" y="136.0" />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
- <id>8962ad5a-0175-1000-ffff-ffffde6db5a6</id>
- <name>RouteOnAttribute</name>
- <position x="-1080.0" y="280.0" />
+ <id>c1318ad1-0c35-3896-b32a-1ccd6c09864a</id>
+ <name>Add source fields and fix timestamp</name>
+ <position x="-2192.0" y="560.0" />
<styles />
<comment />
- <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
+ <class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -3908,217 +3334,584 @@
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Routing Strategy</name>
- <value>Route to Property name</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>94600c6c-704e-3ff8-a2a4-f2f25c71dc3b</value>
+ </property>
+ <property>
+ <name>replacement-value-strategy</name>
+ <value>literal-value</value>
</property>
<property>
- <name>keycloak</name>
- <value>${log_type:equals("keycloak")}</value>
+ <name>/labels/source</name>
+ <value>${tailfile.original.path}</value>
</property>
<property>
- <name>kibana</name>
- <value>${log_type:equals("kibana")}</value>
+ <name>/labels/source_host</name>
+ <value>${hostname()}</value>
</property>
<property>
- <name>elasticsearch</name>
- <value>${log_type:equals("elasticsearch")}</value>
+ <name>/timestamp</name>
+ <value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
</property>
+ </processor>
+ <processor>
+ <id>68663f14-f470-32ee-9cb3-224344d5ad6a</id>
+ <name>UpdateAttribute</name>
+ <position x="-2184.0" y="824.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
<property>
- <name>suricata</name>
- <value>${log_type:equals("suricata")}</value>
+ <name>Delete Attributes Expression</name>
</property>
<property>
- <name>haproxy</name>
- <value>${log_type:equals("haproxy")}</value>
+ <name>Store State</name>
+ <value>Do not store state</value>
</property>
<property>
- <name>mysql</name>
- <value>${log_type:equals("mysql")}</value>
+ <name>Stateful Variables Initial Value</name>
</property>
<property>
- <name>zeek</name>
- <value>${log_type:equals("zeek")}</value>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
</property>
<property>
- <name>nifi</name>
- <value>${log_type:equals("nifi")}</value>
+ <name>data_index</name>
+ <value>nifi-logs</value>
</property>
<property>
- <name>zookeeper</name>
- <value>${log_type:equals("zookeeper")}</value>
+ <name>Authorization</name>
+ <value>Bearer 874rPdPJ6qOSL6HvWcH8xxChqTJxiFtHp94puxh4MygY</value>
</property>
</processor>
- <outputPort>
- <id>bcb879d5-0175-1000-0000-000070879ad0</id>
- <name>To data output</name>
- <position x="-2480.0" y="336.0" />
- <comments />
+ <processor>
+ <id>716d22cb-8b81-342f-abe4-7cdfe2a980ff</id>
+ <name>ConvertRecord</name>
+ <position x="-2200.0" y="264.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ConvertRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
- </outputPort>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>record-reader</name>
+ <value>e3e44ca0-6653-328b-9d3f-b8225312914b</value>
+ </property>
+ <property>
+ <name>record-writer</name>
+ <value>94600c6c-704e-3ff8-a2a4-f2f25c71dc3b</value>
+ </property>
+ <property>
+ <name>include-zero-record-flowfiles</name>
+ <value>true</value>
+ </property>
+ </processor>
+ <processor>
+ <id>609a3835-5317-1c94-ad8f-1d9940869db4</id>
+ <name>TailFile</name>
+ <position x="-2200.0" y="8.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.TailFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>tail-mode</name>
+ <value>Single file</value>
+ </property>
+ <property>
+ <name>File to Tail</name>
+ <value>/opt/nifi/nifi-current/logs/nifi-user.log</value>
+ </property>
+ <property>
+ <name>Rolling Filename Pattern</name>
+ </property>
+ <property>
+ <name>tail-base-directory</name>
+ </property>
+ <property>
+ <name>Initial Start Position</name>
+ <value>Beginning of File</value>
+ </property>
+ <property>
+ <name>File Location</name>
+ <value>Local</value>
+ </property>
+ <property>
+ <name>tailfile-recursive-lookup</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>tailfile-lookup-frequency</name>
+ <value>10 minutes</value>
+ </property>
+ <property>
+ <name>tailfile-maximum-age</name>
+ <value>24 hours</value>
+ </property>
+ </processor>
+ <processor>
+ <id>9ecf3ce4-ae3c-1b06-96f9-5e9c59e51690</id>
+ <name>TailFile</name>
+ <position x="-1784.0" y="8.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.TailFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>tail-mode</name>
+ <value>Single file</value>
+ </property>
+ <property>
+ <name>File to Tail</name>
+ <value>/opt/nifi/nifi-current/logs/nifi-bootstrap.log</value>
+ </property>
+ <property>
+ <name>Rolling Filename Pattern</name>
+ </property>
+ <property>
+ <name>tail-base-directory</name>
+ </property>
+ <property>
+ <name>Initial Start Position</name>
+ <value>Beginning of File</value>
+ </property>
+ <property>
+ <name>File Location</name>
+ <value>Local</value>
+ </property>
+ <property>
+ <name>tailfile-recursive-lookup</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>tailfile-lookup-frequency</name>
+ <value>10 minutes</value>
+ </property>
+ <property>
+ <name>tailfile-maximum-age</name>
+ <value>24 hours</value>
+ </property>
+ </processor>
+ <processor>
+ <id>1e796098-c064-371a-8147-d60b5d41a316</id>
+ <name>TailFile</name>
+ <position x="-2648.0" y="16.0" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.TailFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>tail-mode</name>
+ <value>Single file</value>
+ </property>
+ <property>
+ <name>File to Tail</name>
+ <value>/opt/nifi/nifi-current/logs/nifi-app.log</value>
+ </property>
+ <property>
+ <name>Rolling Filename Pattern</name>
+ </property>
+ <property>
+ <name>tail-base-directory</name>
+ </property>
+ <property>
+ <name>Initial Start Position</name>
+ <value>Beginning of File</value>
+ </property>
+ <property>
+ <name>File Location</name>
+ <value>Local</value>
+ </property>
+ <property>
+ <name>tailfile-recursive-lookup</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>tailfile-lookup-frequency</name>
+ <value>10 minutes</value>
+ </property>
+ <property>
+ <name>tailfile-maximum-age</name>
+ <value>24 hours</value>
+ </property>
+ </processor>
<outputPort>
- <id>349b32fe-a821-1197-0000-00003a0b6fe5</id>
- <name>To enrichment</name>
- <position x="744.0" y="920.0" />
+ <id>1ef39440-1985-3bbb-8e03-859a1c5ee4b1</id>
+ <name>To storage</name>
+ <position x="-2120.0" y="1064.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
- <processGroup>
- <id>89636688-0175-1000-ffff-ffffb1b28a38</id>
- <name>Unknown data</name>
- <position x="-448.0" y="64.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>8963d0f9-0175-1000-0000-000054fbe086</id>
- <name>UpdateAttribute</name>
- <position x="392.0" y="248.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <funnel>
+ <id>beabd3be-6f95-3369-9aa5-4631e6207ec5</id>
+ <position x="-1572.2279720213353" y="320.16022816068823" />
+ </funnel>
+ <connection>
+ <id>0261c39c-d823-3ab3-b680-2d0c2fa152e5</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c1318ad1-0c35-3896-b32a-1ccd6c09864a</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>68663f14-f470-32ee-9cb3-224344d5ad6a</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>214adcbf-0175-1000-ffff-ffffaedeecef</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>9ecf3ce4-ae3c-1b06-96f9-5e9c59e51690</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>0a34d026-0761-3a6d-b261-377d20a40b5d</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c1318ad1-0c35-3896-b32a-1ccd6c09864a</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>beabd3be-6f95-3369-9aa5-4631e6207ec5</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>266ad4f3-c827-337a-b3b5-28bccc8917fc</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>beabd3be-6f95-3369-9aa5-4631e6207ec5</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>0dc021fb-a375-3e48-a5fd-1b9d5c3ad9bd</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>1e796098-c064-371a-8147-d60b5d41a316</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5bc3c82f-bfb5-3e6c-a7bf-141748391beb</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c1318ad1-0c35-3896-b32a-1ccd6c09864a</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>32df885e-4eb1-38fd-bf6b-219725264cf4</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>68663f14-f470-32ee-9cb3-224344d5ad6a</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>1ef39440-1985-3bbb-8e03-859a1c5ee4b1</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>609a3840-5317-1c94-0000-00002897f5ab</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>609a3835-5317-1c94-ad8f-1d9940869db4</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</destinationId>
+ <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <controllerService>
+ <id>e3e44ca0-6653-328b-9d3f-b8225312914b</id>
+ <name>Nifi logs GrokReader</name>
+ <comment />
+ <class>org.apache.nifi.grok.GrokReader</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-record-serialization-services-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <enabled>true</enabled>
+ <property>
+ <name>schema-access-strategy</name>
+ <value>string-fields-from-grok-expression</value>
+ </property>
+ <property>
+ <name>schema-registry</name>
+ </property>
+ <property>
+ <name>schema-name</name>
+ <value>${schema.name}</value>
+ </property>
+ <property>
+ <name>schema-version</name>
+ </property>
+ <property>
+ <name>schema-branch</name>
+ </property>
+ <property>
+ <name>schema-text</name>
+ <value>${avro.schema}</value>
+ </property>
+ <property>
+ <name>Grok Pattern File</name>
+ </property>
+ <property>
+ <name>Grok Expression</name>
+ <value>%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:class} %{GREEDYDATA:message}</value>
+ </property>
+ <property>
+ <name>no-match-behavior</name>
+ <value>append-to-previous-message</value>
+ </property>
+ </controllerService>
+ </processGroup>
+ <processGroup>
+ <id>0c790562-0175-1000-ffff-ffffeaaeafc3</id>
+ <name>FileBeat</name>
+ <position x="-496.0" y="344.0" />
+ <comment />
+ <processGroup>
+ <id>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</id>
+ <name>Suricata</name>
+ <position x="-504.0" y="352.0" />
+ <comment />
+ <processor>
+ <id>19336e9e-3581-3d83-bb51-b9af2f5a6005</id>
+ <name>Filter out DNS events</name>
+ <position x="-371.62446124181497" y="672.8156960893323" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
</bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
+ <maxConcurrentTasks>3</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
<penalizationPeriod>30 sec</penalizationPeriod>
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Delete Attributes Expression</name>
+ <name>Routing Strategy</name>
+ <value>Route to Property name</value>
</property>
<property>
- <name>Store State</name>
- <value>Do not store state</value>
+ <name>dns</name>
+ <value>${event_type:contains("dns")}</value>
</property>
<property>
- <name>Stateful Variables Initial Value</name>
+ <name>flow</name>
+ <value>${event_type:contains("flow")}</value>
</property>
<property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
+ <name>alert</name>
+ <value>${event_type:contains("alert")}</value>
</property>
<property>
- <name>data_index</name>
- <value>logs-filebeat-unknown</value>
+ <name>ssh</name>
+ <value>${event_type:contains("ssh")}</value>
</property>
- </processor>
- <inputPort>
- <id>89639d3d-0175-1000-ffff-ffffb446c257</id>
- <name>Input</name>
- <position x="444.0000243687773" y="80.00000220501622" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>8963b202-0175-1000-0000-000022d64ba2</id>
- <name>Output</name>
- <position x="456.0" y="504.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>8963e649-0175-1000-ffff-fffff03ab629</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>89639d3d-0175-1000-ffff-ffffb446c257</sourceId>
- <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>8963d0f9-0175-1000-0000-000054fbe086</destinationId>
- <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8963f112-0175-1000-0000-00000dfa15b5</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8963d0f9-0175-1000-0000-000054fbe086</sourceId>
- <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8963b202-0175-1000-0000-000022d64ba2</destinationId>
- <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>bc6be78f-0175-1000-ffff-ffffbcd0f569</id>
- <name>NiFi logs</name>
- <position x="-1904.0" y="264.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>bc903708-0175-1000-0000-0000642abebf</id>
- <name>Extract message</name>
- <position x="352.0" y="280.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>tls</name>
+ <value>${event_type:contains("tls")}</value>
</property>
<property>
- <name>record-writer</name>
- <value>bc8e5957-0175-1000-0000-00003346421d</value>
+ <name>http</name>
+ <value>${event_type:contains("http")}</value>
+ </property>
+ <property>
+ <name>stats</name>
+ <value>${event_type:contains("stats")}</value>
+ </property>
+ <property>
+ <name>files</name>
+ <value>${event_type:contains("file")}</value>
</property>
<property>
- <name>include-zero-record-flowfiles</name>
- <value>true</value>
+ <name>smtp</name>
+ <value>${event_type:contains("smtp")}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
+ <autoTerminatedRelationship>smtp</autoTerminatedRelationship>
+ <autoTerminatedRelationship>stats</autoTerminatedRelationship>
+ <autoTerminatedRelationship>unmatched</autoTerminatedRelationship>
</processor>
<processor>
- <id>bc91c66f-0175-1000-0000-00005c7f88ce</id>
- <name>Convert to json</name>
- <position x="1064.0" y="272.0" />
+ <id>d59eabae-f47a-3d88-a1c9-e15c156202d6</id>
+ <name>Extract rrname</name>
+ <position x="-369.29872149802804" y="897.6180433395261" />
<styles />
<comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
+ <class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4126,35 +3919,46 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>record-reader</name>
- <value>bc97858d-0175-1000-0000-0000130a84f8</value>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>record-writer</name>
<value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
- <name>include-zero-record-flowfiles</name>
- <value>true</value>
+ <name>replacement-value-strategy</name>
+ <value>record-path-value</value>
+ </property>
+ <property>
+ <name>/rrname_length</name>
+ <value>/dns/rrname</value>
+ </property>
+ <property>
+ <name>/rrname_domain</name>
+ <value>/dns/rrname</value>
+ </property>
+ <property>
+ <name>/rrname_domain_length</name>
+ <value>/dns/rrname</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
- <id>bcabbf11-0175-1000-0000-000037f4e3d3</id>
- <name>UpdateAttribute</name>
- <position x="1072.0" y="472.0" />
+ <id>24e1d8ed-10f4-3b46-958c-f2fb676e3192</id>
+ <name>Remove unnecessary filebeat fields</name>
+ <position x="-987.5658863682004" y="234.96963460665665" />
<styles />
<comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <class>org.apache.nifi.processors.standard.JoltTransformJSON</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4162,40 +3966,59 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Delete Attributes Expression</name>
+ <name>jolt-transform</name>
+ <value>jolt-transform-chain</value>
</property>
<property>
- <name>Store State</name>
- <value>Do not store state</value>
+ <name>jolt-custom-class</name>
</property>
<property>
- <name>Stateful Variables Initial Value</name>
+ <name>jolt-custom-modules</name>
</property>
<property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
+ <name>jolt-spec</name>
+ <value>[{
+ "operation": "shift",
+ "spec": {
+ "*": {
+ "json": {
+ "*": {
+ "@": "[#4].&"
+ }
+ },
+ "host": {
+ "name": "[#3].beat_host"
+ },
+ "source": "[#2].source"
+ }
+ }
+}]</value>
</property>
<property>
- <name>data_index</name>
- <value>logs-nifi</value>
+ <name>Transform Cache Size</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>pretty_print</name>
+ <value>false</value>
</property>
</processor>
<processor>
- <id>bc9ffeb0-0175-1000-0000-00000a88d684</id>
- <name>UpdateRecord</name>
- <position x="352.0" y="472.0" />
+ <id>47757d9f-c23d-33ca-9c88-3c8722bd00a5</id>
+ <name>Add rrname_domain++</name>
+ <position x="-368.27336608185624" y="1138.9635842383886" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4203,7 +4026,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
@@ -4220,152 +4043,45 @@
<value>literal-value</value>
</property>
<property>
- <name>/labels/source_file</name>
- <value>${source_file}</value>
+ <name>/rrname_length</name>
+ <value>${field.value:length():toNumber()}</value>
</property>
<property>
- <name>/labels/source_host</name>
- <value>${source_host}</value>
+ <name>/ip_src_addr</name>
+ <value>0.0.0.0</value>
</property>
<property>
- <name>/timestamp</name>
- <value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
+ <name>/src_ip</name>
+ <value>0.0.0.0</value>
+ </property>
+ <property>
+ <name>/ip_dst_addr</name>
+ <value>0.0.0.0</value>
+ </property>
+ <property>
+ <name>/rrname_domain</name>
+ <value>${field.value:substringBeforeLast('.'):substringAfterLast('.'):append(${field.value:substringAfterLast('.'):prepend('.')})}</value>
+ </property>
+ <property>
+ <name>/dest_ip</name>
+ <value>0.0.0.0</value>
+ </property>
+ <property>
+ <name>/rrname_domain_length</name>
+ <value>${field.value:substringBeforeLast('.'):substringAfterLast('.'):append(${field.value:substringAfterLast('.'):prepend('.')}):length():toNumber()}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
- <inputPort>
- <id>bc6c2159-0175-1000-ffff-ffffb4de4d47</id>
- <name>Input</name>
- <position x="397.9999517774115" y="110.99999315685733" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>bca9636a-0175-1000-0000-000013fa95aa</id>
- <name>Output</name>
- <position x="1120.0" y="808.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>bc90c7ac-0175-1000-ffff-fffffa80b534</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bc903708-0175-1000-0000-0000642abebf</sourceId>
- <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bc91c66f-0175-1000-0000-00005c7f88ce</destinationId>
- <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bc924694-0175-1000-0000-00005b0604b6</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bc91c66f-0175-1000-0000-00005c7f88ce</sourceId>
- <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bc9ffeb0-0175-1000-0000-00000a88d684</destinationId>
- <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bc6e1b20-0175-1000-ffff-ffff9e7dcb75</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bc6c2159-0175-1000-ffff-ffffb4de4d47</sourceId>
- <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>bc903708-0175-1000-0000-0000642abebf</destinationId>
- <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bcad2e36-0175-1000-0000-00002b6e8fe7</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bcabbf11-0175-1000-0000-000037f4e3d3</sourceId>
- <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bca9636a-0175-1000-0000-000013fa95aa</destinationId>
- <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bca97855-0175-1000-ffff-ffffbd18cb66</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bc9ffeb0-0175-1000-0000-00000a88d684</sourceId>
- <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bcabbf11-0175-1000-0000-000037f4e3d3</destinationId>
- <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>895eab20-0175-1000-0000-00007e13267d</id>
- <name>Common ListenBeats</name>
- <position x="-1096.0" y="0.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
- <id>71be315f-7e16-1cce-89f1-d5bd502f889f</id>
+ <id>e89b0470-bff2-323c-92e5-5fb2d3949070</id>
<name>Prepend [</name>
- <position x="-1086.1517800521056" y="160.65881341602864" />
+ <position x="-996.7179797450688" y="47.535499055489254" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.ReplaceText</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4373,7 +4089,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
@@ -4391,7 +4107,7 @@
</property>
<property>
<name>Maximum Buffer Size</name>
- <value>2 MB</value>
+ <value>1 MB</value>
</property>
<property>
<name>Replacement Strategy</name>
@@ -4405,19 +4121,18 @@
<name>Line-by-Line Evaluation Mode</name>
<value>All</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
- <id>d3e43667-10ef-1528-b935-47c2f077f2c9</id>
- <name>ListenBeats</name>
- <position x="-616.0" y="-56.0" />
+ <id>fd6b6513-51f8-3a96-a764-13bd39ec7f84</id>
+ <name>Partition records based on event_type</name>
+ <position x="-382.59400260581754" y="446.9900134408068" />
<styles />
<comment />
- <class>org.apache.nifi.processors.beats.ListenBeats</class>
+ <class>org.apache.nifi.processors.standard.PartitionRecord</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-beats-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4425,65 +4140,35 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Local Network Interface</name>
- </property>
- <property>
- <name>Port</name>
- <value>6001</value>
- </property>
- <property>
- <name>Receive Buffer Size</name>
- <value>65507 B</value>
- </property>
- <property>
- <name>Max Size of Message Queue</name>
- <value>10000</value>
- </property>
- <property>
- <name>Max Size of Socket Buffer</name>
- <value>2 MB</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>Max Batch Size</name>
- <value>10000</value>
- </property>
- <property>
- <name>Message Delimiter</name>
- <value>,\n</value>
- </property>
- <property>
- <name>Max Number of TCP Connections</name>
- <value>100</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
- <name>SSL_CONTEXT_SERVICE</name>
- <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
- <name>Client Auth</name>
- <value>NONE</value>
+ <name>event_type</name>
+ <value>/event_type</value>
</property>
+ <autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
<processor>
- <id>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</id>
- <name>Append ]</name>
- <position x="-424.0" y="160.0" />
+ <id>a88dab36-f543-32fc-8f45-aa11b99c0ff4</id>
+ <name>Add standardized fields</name>
+ <position x="-982.3277701858627" y="433.7665258942376" />
<styles />
<comment />
- <class>org.apache.nifi.processors.standard.ReplaceText</class>
+ <class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4491,51 +4176,50 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Regular Expression</name>
- <value>(?s)(^.*$)</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
- <name>Replacement Value</name>
- <value>]</value>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
- <name>Character Set</name>
- <value>UTF-8</value>
+ <name>replacement-value-strategy</name>
+ <value>record-path-value</value>
</property>
<property>
- <name>Maximum Buffer Size</name>
- <value>2 MB</value>
+ <name>/ip_dst_port</name>
+ <value>/dest_port</value>
</property>
<property>
- <name>Replacement Strategy</name>
- <value>Append</value>
+ <name>/ip_src_addr</name>
+ <value>/src_ip</value>
</property>
<property>
- <name>Evaluation Mode</name>
- <value>Entire text</value>
+ <name>/ip_dst_addr</name>
+ <value>/dest_ip</value>
</property>
<property>
- <name>Line-by-Line Evaluation Mode</name>
- <value>All</value>
+ <name>/ip_src_port</name>
+ <value>/src_port</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
- <id>d64f3acd-54a6-1b39-b1af-cc0a26156d5b</id>
- <name>ListenBeats</name>
- <position x="-1076.9243538376497" y="-51.550721133258094" />
+ <id>1a038948-9e9a-3523-b899-990077bfd575</id>
+ <name>Convert timestamp</name>
+ <position x="-385.7461824498648" y="233.13395543765722" />
<styles />
<comment />
- <class>org.apache.nifi.processors.beats.ListenBeats</class>
+ <class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-beats-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4543,64 +4227,46 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Local Network Interface</name>
- </property>
- <property>
- <name>Port</name>
- <value>6000</value>
- </property>
- <property>
- <name>Receive Buffer Size</name>
- <value>1024kb</value>
- </property>
- <property>
- <name>Max Size of Message Queue</name>
- <value>10000</value>
- </property>
- <property>
- <name>Max Size of Socket Buffer</name>
- <value>4 MB</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
+ <name>record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
- <name>Max Batch Size</name>
- <value>500</value>
+ <name>record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
- <name>Message Delimiter</name>
- <value>,\n</value>
+ <name>replacement-value-strategy</name>
+ <value>literal-value</value>
</property>
<property>
- <name>Max Number of TCP Connections</name>
- <value>200</value>
+ <name>/TLP</name>
+ <value>AMBER</value>
</property>
<property>
- <name>SSL_CONTEXT_SERVICE</name>
+ <name>/mime.type</name>
+ <value>application/json</value>
</property>
<property>
- <name>Client Auth</name>
- <value>NONE</value>
+ <name>/timestamp</name>
+ <value>${field.value:replaceFirst('\+(\d\d)(\d\d)','+$1:$2')}</value>
</property>
</processor>
<processor>
- <id>bb719fee-0175-1000-ffff-ffffb73dd31a</id>
- <name>Rename @ fields</name>
- <position x="-1080.0" y="360.0" />
+ <id>7eeb23aa-b112-3cc8-bb56-2ca20b456907</id>
+ <name>Add enrichment attributes</name>
+ <position x="-381.71987132795925" y="1362.5536493927905" />
<styles />
<comment />
- <class>org.apache.nifi.processors.standard.JoltTransformJSON</class>
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4608,54 +4274,170 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>jolt-transform</name>
- <value>jolt-transform-chain</value>
+ <name>Delete Attributes Expression</name>
</property>
<property>
- <name>jolt-custom-class</name>
+ <name>Store State</name>
+ <value>Do not store state</value>
</property>
<property>
- <name>jolt-custom-modules</name>
+ <name>Stateful Variables Initial Value</name>
</property>
<property>
- <name>jolt-spec</name>
- <value>[{
- "operation": "shift",
- "spec": {
- "*": {
- "\\@timestamp":"[&1].timestamp",
- "\\@metadata":"[&1].metadata",
- "*": "[&1].&"
- }
- }
-}]</value>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
</property>
<property>
- <name>Transform Cache Size</name>
- <value>1</value>
+ <name>enrich_domain1</name>
+ <value>/rrname_domain</value>
</property>
<property>
- <name>pretty_print</name>
- <value>false</value>
+ <name>enrich_fqdn1</name>
+ <value>/dns/rrname</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
- <id>896047e7-0175-1000-ffff-ffffc69204e4</id>
- <name>PartitionRecord</name>
- <position x="-424.0" y="368.0" />
+ <id>46cdd7aa-91f0-307c-90aa-65747e558f25</id>
+ <name>Add attributes</name>
+ <position x="-990.1194195007834" y="665.1839855126569" />
<styles />
<comment />
- <class>org.apache.nifi.processors.standard.PartitionRecord</class>
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>data_id</name>
+ <value>suricata-${event_type}</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-${beats.sender:substringBeforeLast('/'):substringBeforeLast('.'):substringAfterLast('.')}-suricata-${event_type}</value>
+ </property>
+ <property>
+ <name>data_type</name>
+ <value>suricata</value>
+ </property>
+ <property>
+ <name>enrich_ip2</name>
+ <value>/ip_dst_addr</value>
+ </property>
+ <property>
+ <name>enrich_ip1</name>
+ <value>/ip_src_addr</value>
+ </property>
+ <property>
+ <name>TLP</name>
+ <value>AMBER</value>
+ </property>
+ </processor>
+ <processor>
+ <id>2b0f0d27-a69e-30c9-b3a6-1499ff955a30</id>
+ <name>ListenBeats</name>
+ <position x="-987.897149146032" y="-167.11494242687536" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.beats.ListenBeats</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-beats-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>2</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>NONE</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Local Network Interface</name>
+ </property>
+ <property>
+ <name>Port</name>
+ <value>6101</value>
+ </property>
+ <property>
+ <name>Receive Buffer Size</name>
+ <value>65507 B</value>
+ </property>
+ <property>
+ <name>Max Size of Message Queue</name>
+ <value>100000</value>
+ </property>
+ <property>
+ <name>Max Size of Socket Buffer</name>
+ <value>10 MB</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Max Batch Size</name>
+ <value>5000</value>
+ </property>
+ <property>
+ <name>Message Delimiter</name>
+ <value>,\n</value>
+ </property>
+ <property>
+ <name>Max Number of TCP Connections</name>
+ <value>90</value>
+ </property>
+ <property>
+ <name>SSL_CONTEXT_SERVICE</name>
+ </property>
+ <property>
+ <name>Client Auth</name>
+ <value>REQUIRED</value>
+ </property>
+ </processor>
+ <processor>
+ <id>be18c77e-5e4e-3552-ac9b-892ab69a9d49</id>
+ <name>Append ]</name>
+ <position x="-396.7049152015222" y="50.4274414148523" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ReplaceText</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4663,52 +4445,70 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>Regular Expression</name>
+ <value>(?s)(^.*$)</value>
</property>
<property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ <name>Replacement Value</name>
+ <value>]</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
</property>
<property>
- <name>log_type</name>
- <value>/fields/log_type</value>
+ <name>Maximum Buffer Size</name>
+ <value>1 MB</value>
</property>
<property>
- <name>source_host</name>
- <value>/host/name</value>
+ <name>Replacement Strategy</name>
+ <value>Append</value>
</property>
<property>
- <name>source_file</name>
- <value>/log/file/path</value>
+ <name>Evaluation Mode</name>
+ <value>Entire text</value>
+ </property>
+ <property>
+ <name>Line-by-Line Evaluation Mode</name>
+ <value>All</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
<outputPort>
- <id>89620b1a-0175-1000-0000-000078566f34</id>
- <name>Output</name>
- <position x="-360.0" y="704.0" />
+ <id>055308a4-d020-39a9-9da4-b165796ef717</id>
+ <name>To enrichment</name>
+ <position x="-900.0911671813442" y="1418.3104443450675" />
<comments />
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
</outputPort>
+ <funnel>
+ <id>d8f19295-5666-31a8-b701-52214c4db51d</id>
+ <position x="-1500.995244929405" y="257.20806784146276" />
+ </funnel>
+ <funnel>
+ <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
+ <position x="283.72871497338747" y="273.4623850295515" />
+ </funnel>
+ <funnel>
+ <id>c4afa3d5-0170-1000-ffff-ffffe437a306</id>
+ <position x="396.10723355029654" y="1188.222598705122" />
+ </funnel>
<connection>
- <id>bb8aafca-0175-1000-0000-000038f8e9fc</id>
+ <id>d39ff93b-85e9-3c56-9f44-1916d1abcd9d</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>896047e7-0175-1000-ffff-ffffc69204e4</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+ <sourceId>a88dab36-f543-32fc-8f45-aa11b99c0ff4</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>89620b1a-0175-1000-0000-000078566f34</destinationId>
- <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
+ <destinationId>fd6b6513-51f8-3a96-a764-13bd39ec7f84</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
@@ -4718,18 +4518,18 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>895f18a7-0175-1000-ffff-ffffbc2237fd</id>
+ <id>c4ae2f82-0170-1000-ffff-ffff91d33f16</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>71be315f-7e16-1cce-89f1-d5bd502f889f</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+ <sourceId>19336e9e-3581-3d83-bb51-b9af2f5a6005</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</destinationId>
- <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
+ <destinationId>d59eabae-f47a-3d88-a1c9-e15c156202d6</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
+ <relationship>dns</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -4738,16 +4538,16 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>895ee440-0175-1000-ffff-ffffd3ff3143</id>
+ <id>a4471b0c-c924-31e0-9aa1-7cf56b1be0ed</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>d64f3acd-54a6-1b39-b1af-cc0a26156d5b</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+ <sourceId>be18c77e-5e4e-3552-ac9b-892ab69a9d49</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId>
- <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
+ <destinationId>24e1d8ed-10f4-3b46-958c-f2fb676e3192</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
@@ -4758,16 +4558,16 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>ac7a988a-0175-1000-ffff-ffff86c66751</id>
+ <id>f9a8aee6-502f-3eb9-8806-8964276d4ca0</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>d3e43667-10ef-1528-b935-47c2f077f2c9</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+ <sourceId>24e1d8ed-10f4-3b46-958c-f2fb676e3192</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId>
- <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
+ <destinationId>1a038948-9e9a-3523-b899-990077bfd575</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
@@ -4778,36 +4578,63 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>bb8a6c6c-0175-1000-0000-00000abdc8f9</id>
+ <id>e8ad07a6-cd62-3473-9b16-833cf43026a6</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>bb719fee-0175-1000-ffff-ffffb73dd31a</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+ <sourceId>2b0f0d27-a69e-30c9-b3a6-1499ff955a30</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>896047e7-0175-1000-ffff-ffffc69204e4</destinationId>
- <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
+ <destinationId>e89b0470-bff2-323c-92e5-5fb2d3949070</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>1 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>33baee02-9354-3b7f-a910-6220a5f6108f</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-775.4788208007812" y="947.9116821289062" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>19336e9e-3581-3d83-bb51-b9af2f5a6005</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>055308a4-d020-39a9-9da4-b165796ef717</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>alert</relationship>
+ <relationship>files</relationship>
+ <relationship>http</relationship>
+ <relationship>ssh</relationship>
+ <relationship>tls</relationship>
+ <relationship>flow</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>8961779d-0175-1000-0000-00003ef237de</id>
+ <id>c010a48c-a3af-3cfc-9693-9885925e763e</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
+ <sourceId>fd6b6513-51f8-3a96-a764-13bd39ec7f84</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
- <destinationId>bb719fee-0175-1000-ffff-ffffb73dd31a</destinationId>
- <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId>
+ <destinationId>46cdd7aa-91f0-307c-90aa-65747e558f25</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>success</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
@@ -4817,130 +4644,368 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
- </processGroup>
- <processGroup>
- <id>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</id>
- <name>Suricata</name>
- <position x="-448.0" y="264.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>8d1bef35-0175-1000-0000-0000746fa33d</id>
- <name>RouteOnAttribute</name>
- <position x="-984.0" y="640.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Routing Strategy</name>
- <value>Route to Property name</value>
- </property>
- <property>
- <name>dns</name>
- <value>${event_type:equals("dns")}</value>
- </property>
- <property>
- <name>tls</name>
- <value>${event_type:equals("tls")}</value>
- </property>
- </processor>
- <processor>
- <id>24e1d8ed-10f4-3b46-958c-f2fb676e3192</id>
- <name>Normalize fields</name>
- <position x="-987.5658863682004" y="234.96963460665665" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.JoltTransformJSON</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>jolt-transform</name>
- <value>jolt-transform-chain</value>
- </property>
- <property>
- <name>jolt-custom-class</name>
- </property>
- <property>
- <name>jolt-custom-modules</name>
+ <connection>
+ <id>3a82b9ea-a974-3750-ad78-275da67285e6</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>be18c77e-5e4e-3552-ac9b-892ab69a9d49</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c4afb718-0170-1000-0000-000061284251</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>d59eabae-f47a-3d88-a1c9-e15c156202d6</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c4afa3d5-0170-1000-ffff-ffffe437a306</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>67789d5d-ebdc-390c-adc8-f2111f467ad4</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>fd6b6513-51f8-3a96-a764-13bd39ec7f84</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c4afc7bc-0170-1000-ffff-ffffae3762dd</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>47757d9f-c23d-33ca-9c88-3c8722bd00a5</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c4afa3d5-0170-1000-ffff-ffffe437a306</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>f4bd2bed-88a1-396f-974b-19dcb5f40101</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e89b0470-bff2-323c-92e5-5fb2d3949070</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>d8f19295-5666-31a8-b701-52214c4db51d</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>9b860d17-8918-3956-a8b2-54ec49231c37</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>47757d9f-c23d-33ca-9c88-3c8722bd00a5</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>7eeb23aa-b112-3cc8-bb56-2ca20b456907</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>98594ca0-2fce-349c-8432-94f4d021d1fe</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e89b0470-bff2-323c-92e5-5fb2d3949070</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>be18c77e-5e4e-3552-ac9b-892ab69a9d49</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>d7ef534a-9fb6-3973-b2fa-2738705db47a</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>24e1d8ed-10f4-3b46-958c-f2fb676e3192</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>d8f19295-5666-31a8-b701-52214c4db51d</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c31d92b9-0e34-387d-86df-9536bf2ed9c9</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>1a038948-9e9a-3523-b899-990077bfd575</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>a88dab36-f543-32fc-8f45-aa11b99c0ff4</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>7417695b-cbde-3637-bb24-2e265bb2817c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>46cdd7aa-91f0-307c-90aa-65747e558f25</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>19336e9e-3581-3d83-bb51-b9af2f5a6005</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>3e181820-b214-399d-a0df-474d15e2f146</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>1a038948-9e9a-3523-b899-990077bfd575</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>e9962c43-9689-39b9-a1ba-cd2eac598802</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>a88dab36-f543-32fc-8f45-aa11b99c0ff4</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>d8f19295-5666-31a8-b701-52214c4db51d</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>1f219054-ea65-3700-a503-2d24acf2c754</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>7eeb23aa-b112-3cc8-bb56-2ca20b456907</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>055308a4-d020-39a9-9da4-b165796ef717</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>8c6c7a60-0856-3a39-8ed6-6e7d0b98c0ae</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>d59eabae-f47a-3d88-a1c9-e15c156202d6</sourceId>
+ <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>47757d9f-c23d-33ca-9c88-3c8722bd00a5</destinationId>
+ <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <processGroup>
+ <id>83691174-683f-3c7c-8526-8fc00397aee1</id>
+ <name>Zeek</name>
+ <position x="-504.0" y="152.0" />
+ <comment />
+ <processor>
+ <id>39492e6c-faf0-3bfa-bd16-51a1f8be4c71</id>
+ <name>ListenBeats</name>
+ <position x="-1114.9155421491096" y="263.01449694104195" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.beats.ListenBeats</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-beats-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Local Network Interface</name>
</property>
<property>
- <name>jolt-spec</name>
- <value>[{
- "operation": "shift",
- "spec": {
- "*": {
- "json": {
- "*": "[&2].&"
- },
- "host": {
- "name": "[&2].labels.source_host"
- },
- "source": "[&1].labels.source"
- }
- }
-}, {
- "operation": "shift",
- "spec": {
- "*": {
- "dest_ip":"[&1].destination.ip",
- "dest_port":"[&1].destination.port",
- "src_ip":"[&1].source.ip",
- "src_port":"[&1].source.port",
- "*": "[&1].&"
- }
- }
-}]</value>
+ <name>Port</name>
+ <value>6100</value>
</property>
<property>
- <name>Transform Cache Size</name>
- <value>1</value>
+ <name>Receive Buffer Size</name>
+ <value>65507 B</value>
</property>
<property>
- <name>pretty_print</name>
- <value>false</value>
+ <name>Max Size of Message Queue</name>
+ <value>10000</value>
+ </property>
+ <property>
+ <name>Max Size of Socket Buffer</name>
+ <value>2 MB</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Max Batch Size</name>
+ <value>10000</value>
+ </property>
+ <property>
+ <name>Message Delimiter</name>
+ <value>,\n</value>
+ </property>
+ <property>
+ <name>Max Number of TCP Connections</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>SSL_CONTEXT_SERVICE</name>
+ </property>
+ <property>
+ <name>Client Auth</name>
+ <value>REQUIRED</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
- <id>fd6b6513-51f8-3a96-a764-13bd39ec7f84</id>
- <name>Partition records based on event_type</name>
- <position x="-382.59400260581754" y="446.9900134408068" />
+ <id>ac17155e-32f1-3be8-843e-00877c210519</id>
+ <name>Prepend [</name>
+ <position x="-1124.1429683635654" y="475.2240314903287" />
<styles />
<comment />
- <class>org.apache.nifi.processors.standard.PartitionRecord</class>
+ <class>org.apache.nifi.processors.standard.ReplaceText</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4948,36 +5013,107 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ <name>Regular Expression</name>
+ <value>(?s)(^.*$)</value>
</property>
<property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
+ <name>Replacement Value</name>
+ <value>[</value>
</property>
<property>
- <name>event_type</name>
- <value>/event_type</value>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Maximum Buffer Size</name>
+ <value>1 MB</value>
+ </property>
+ <property>
+ <name>Replacement Strategy</name>
+ <value>Prepend</value>
+ </property>
+ <property>
+ <name>Evaluation Mode</name>
+ <value>Entire text</value>
+ </property>
+ <property>
+ <name>Line-by-Line Evaluation Mode</name>
+ <value>All</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
</processor>
<processor>
- <id>1a038948-9e9a-3523-b899-990077bfd575</id>
- <name>Convert timestamp</name>
- <position x="-385.7461824498648" y="233.13395543765722" />
+ <id>fec43039-de5d-1e3b-850a-5e25d7b93c76</id>
+ <name>UpdateAttribute</name>
+ <position x="-1121.584644408096" y="913.629598069974" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-uninett-darknet</value>
+ </property>
+ <property>
+ <name>mime.type</name>
+ <value>application/json</value>
+ </property>
+ <property>
+ <name>enrich_ip2</name>
+ <value>/ip_dst_addr</value>
+ </property>
+ <property>
+ <name>enrich_ip1</name>
+ <value>/ip_src_addr</value>
+ </property>
+ <property>
+ <name>TLP</name>
+ <value>GREEN</value>
+ </property>
+ </processor>
+ <processor>
+ <id>9027e415-c8cd-355e-af16-0c635f43832f</id>
+ <name>Convert timestamp and add fields</name>
+ <position x="-472.77987807459795" y="696.7530680701591" />
<styles />
<comment />
<class>org.apache.nifi.processors.standard.UpdateRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -4985,7 +5121,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
@@ -5003,29 +5139,40 @@
</property>
<property>
<name>/TLP</name>
- <value>AMBER</value>
+ <value>GREEN</value>
+ </property>
+ <property>
+ <name>/data_type</name>
+ <value>zeek</value>
+ </property>
+ <property>
+ <name>/data_index</name>
+ <value>logs-zeek-conn</value>
</property>
<property>
<name>/mime.type</name>
<value>application/json</value>
</property>
+ <property>
+ <name>/data_id</name>
+ <value>zeek_conn</value>
+ </property>
<property>
<name>/timestamp</name>
- <value>${field.value:replaceFirst('\+(\d\d)(\d\d)','+$1:$2')}</value>
+ <value>${field.value:multiply(1000):format('yyyy-MM-dd HH:mm:ss.SSSZ'):replace(' ','T'):replaceFirst('\+(\d\d)(\d\d)','+$1:$2')}</value>
</property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
</processor>
<processor>
- <id>46cdd7aa-91f0-307c-90aa-65747e558f25</id>
- <name>Add attributes</name>
- <position x="-984.0" y="456.0" />
+ <id>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</id>
+ <name>Remove filebeat fields</name>
+ <position x="-1121.1565561587029" y="706.9002449806696" />
<styles />
<comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <class>org.apache.nifi.processors.jolt.record.JoltTransformRecord</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-jolt-record-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriod>0 sec</schedulingPeriod>
@@ -5033,5925 +5180,588 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
- <name>Delete Attributes Expression</name>
+ <name>jolt-record-record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
- <name>Store State</name>
- <value>Do not store state</value>
+ <name>jolt-record-record-writer</name>
+ <value>17b30955-5464-3709-8a32-69a459850cfa</value>
</property>
<property>
- <name>Stateful Variables Initial Value</name>
+ <name>jolt-record-transform</name>
+ <value>jolt-transform-chain</value>
</property>
<property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
+ <name>jolt-record-custom-class</name>
</property>
<property>
- <name>data_id</name>
- <value>suricata-${event_type}</value>
+ <name>jolt-record-custom-modules</name>
</property>
<property>
- <name>data_index</name>
- <value>logs-${beats.sender:substringBeforeLast('/'):substringBeforeLast('.'):substringAfterLast('.')}-suricata-${event_type}</value>
+ <name>jolt-record-spec</name>
+ <value>[
+ {
+ "operation": "shift",
+ "spec": {
+ "json": {
+ "ts": "timestamp",
+ "*": {
+ "@": "&"
+ }
+ },
+ "host": {
+ "name": "host"
+ },
+ "source": "source"
+ }
+ },{
+ "operation" : "modify-overwrite-beta",
+ "spec" :
+ {
+ "timestamp": "=toString"
+ }
+ }
+]</value>
</property>
<property>
- <name>data_type</name>
- <value>suricata</value>
+ <name>jolt-record-transform-cache-size</name>
+ <value>1</value>
</property>
+ <autoTerminatedRelationship>original</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>06501f48-82c7-3c36-b99c-7368a322608b</id>
+ <name>Append ]</name>
+ <position x="-465.8393574027825" y="478.1159738496917" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ReplaceText</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
<property>
- <name>enrich_ip2</name>
- <value>/destination/ip</value>
+ <name>Regular Expression</name>
+ <value>(?s)(^.*$)</value>
</property>
<property>
- <name>enrich_ip1</name>
- <value>/source/ip</value>
+ <name>Replacement Value</name>
+ <value>]</value>
</property>
<property>
- <name>TLP</name>
- <value>AMBER</value>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Maximum Buffer Size</name>
+ <value>1 MB</value>
+ </property>
+ <property>
+ <name>Replacement Strategy</name>
+ <value>Append</value>
+ </property>
+ <property>
+ <name>Evaluation Mode</name>
+ <value>Entire text</value>
+ </property>
+ <property>
+ <name>Line-by-Line Evaluation Mode</name>
+ <value>All</value>
</property>
</processor>
- <inputPort>
- <id>8d13c952-0175-1000-0000-00007e8f4cae</id>
- <name>Input</name>
- <position x="-928.0" y="16.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
<outputPort>
- <id>055308a4-d020-39a9-9da4-b165796ef717</id>
+ <id>a28a9e95-1003-3ea6-9af6-a334c1aec07c</id>
<name>To enrichment</name>
- <position x="-928.0" y="1208.0" />
+ <position x="-1065.7090714972117" y="1164.8389289189608" />
<comments />
- <scheduledState>RUNNING</scheduledState>
+ <scheduledState>STOPPED</scheduledState>
</outputPort>
- <processGroup>
- <id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id>
- <name>TLS events</name>
- <position x="-384.0" y="872.0" />
+ <funnel>
+ <id>06521038-335b-3139-839d-ab43a013ce03</id>
+ <position x="-1557.869726298236" y="758.8984861527665" />
+ </funnel>
+ <funnel>
+ <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id>
+ <position x="248.5321508445502" y="703.4412774751572" />
+ </funnel>
+ <connection>
+ <id>216d4dcf-f425-33d0-a5c1-5cdf1402162e</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>06501f48-82c7-3c36-b99c-7368a322608b</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>3c739604-b69c-3e86-ba4c-a4739078837c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>9027e415-c8cd-355e-af16-0c635f43832f</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c8c0a13d-0170-1000-ffff-ffff874141fa</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>295b97b9-1291-3a83-8191-78a300d0feaa</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>39492e6c-faf0-3bfa-bd16-51a1f8be4c71</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>ac17155e-32f1-3be8-843e-00877c210519</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c8be8213-0170-1000-0000-0000695bc36c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>fec43039-de5d-1e3b-850a-5e25d7b93c76</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>a056b363-8398-3877-8750-1bc9dcb9b1cd</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>ac17155e-32f1-3be8-843e-00877c210519</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>06501f48-82c7-3c36-b99c-7368a322608b</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>ee8556df-9826-3d45-82de-5c1c876db435</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>06521038-335b-3139-839d-ab43a013ce03</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>2d0f222e-d08e-31fd-b5e1-1ce178368e4c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>9027e415-c8cd-355e-af16-0c635f43832f</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>ed8609a1-bd09-391e-831b-1ab5b53a5049</id>
+ <name />
+ <bendPoints />
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>06501f48-82c7-3c36-b99c-7368a322608b</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c8c0a13d-0170-1000-ffff-ffff874141fa</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c8be6e60-0170-1000-ffff-ffffe34d52ef</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>9027e415-c8cd-355e-af16-0c635f43832f</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>fec43039-de5d-1e3b-850a-5e25d7b93c76</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>2a6e865b-4b36-3807-8bd7-eb2f39f95d4f</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>ac17155e-32f1-3be8-843e-00877c210519</sourceId>
+ <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>06521038-335b-3139-839d-ab43a013ce03</destinationId>
+ <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ </processGroup>
+ <processGroup>
+ <id>b3d57504-7c06-37a3-b59b-8723f60fa728</id>
+ <name>Test data</name>
+ <position x="-496.0" y="552.0" />
+ <comment />
+ <outputPort>
+ <id>d30dc946-251a-307c-8e88-f2262b0bb194</id>
+ <name>To enrichment</name>
+ <position x="731.0454088698874" y="433.2315817172085" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </outputPort>
+ <processGroup>
+ <id>0c83ef26-0175-1000-ffff-ffffcac37910</id>
+ <name>Suricata</name>
+ <position x="462.0553417896858" y="119.99261716112323" />
+ <comment />
+ <processor>
+ <id>bb7dc9ff-2d25-3134-9617-cca3cabe9179</id>
+ <name>Alerts</name>
+ <position x="496.0" y="392.0" />
+ <styles />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>9279850b-0175-1000-0000-00001e74d182</id>
- <name>Copy SNI</name>
- <position x="504.0" y="320.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>record-path-value</value>
- </property>
- <property>
- <name>/tls/sni_length</name>
- <value>/tls/sni</value>
- </property>
- <property>
- <name>/tls/sni_domain_length</name>
- <value>/tls/sni</value>
- </property>
- <property>
- <name>/tls/sni_domain</name>
- <value>/tls/sni</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>349b3279-a821-1197-aaa6-7e5472dccbef</id>
- <name>Add sni_domain ++</name>
- <position x="504.0" y="544.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/tls/sni_length</name>
- <value>${field.value:length():toNumber()}</value>
- </property>
- <property>
- <name>/tls/sni_domain_length</name>
- <value>${field.value:substringBeforeLast('.'):substringAfterLast('.'):append(${field.value:substringAfterLast('.'):prepend('.')}):length():toNumber()}</value>
- </property>
- <property>
- <name>/tls/sni_domain</name>
- <value>${field.value:substringBeforeLast('.'):substringAfterLast('.'):append(${field.value:substringAfterLast('.'):prepend('.')})}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>349b3291-a821-1197-0000-000032560c6a</id>
- <name>Specify enrichment fields</name>
- <position x="504.0" y="752.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>enrich_domain1</name>
- <value>/tls/sni_domain</value>
- </property>
- <property>
- <name>enrich_fqdn1</name>
- <value>/tls/sni</value>
- </property>
- </processor>
- <inputPort>
- <id>92795a59-0175-1000-ffff-ffff89bc5f21</id>
- <name>Input</name>
- <position x="552.9999060626994" y="144.00001181679164" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>349b32d8-a821-1197-0000-000025a75a3b</id>
- <name>Output</name>
- <position x="552.0" y="976.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>9279996e-0175-1000-0000-000037fbed8b</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>92795a59-0175-1000-ffff-ffff89bc5f21</sourceId>
- <sourceGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>9279850b-0175-1000-0000-00001e74d182</destinationId>
- <destinationGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b3297-a821-1197-0000-0000717807b6</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>349b3279-a821-1197-aaa6-7e5472dccbef</sourceId>
- <sourceGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>349b3291-a821-1197-0000-000032560c6a</destinationId>
- <destinationGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b327f-a821-1197-ffff-ffff8946a863</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>9279850b-0175-1000-0000-00001e74d182</sourceId>
- <sourceGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>349b3279-a821-1197-aaa6-7e5472dccbef</destinationId>
- <destinationGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b32da-a821-1197-0000-000047979e25</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>349b3291-a821-1197-0000-000032560c6a</sourceId>
- <sourceGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>349b32d8-a821-1197-0000-000025a75a3b</destinationId>
- <destinationGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>8d1ad21f-0175-1000-0000-00003c540411</id>
- <name>DNS events</name>
- <position x="-1000.0" y="872.0" />
+ <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>10 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>File Size</name>
+ <value>0B</value>
+ </property>
+ <property>
+ <name>Batch Size</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>Data Format</name>
+ <value>Text</value>
+ </property>
+ <property>
+ <name>Unique FlowFiles</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>generate-ff-custom-text</name>
+ <value>[{"stream": 0,"flow": {"bytes_toserver": 74,"bytes_toclient": 0,"start": "${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","pkts_toserver": 1,"pkts_toclient": 0},"vlan": 665,"ip_dst_port": 54323,"in_iface": "ens1f3","payload": "","timestamp": "${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","proto": "TCP","event_type": "alert","alert": {"category": "Not Suspicious Traffic","severity": 3,"action": "allowed","gid": 1,"signature_id": 29999991,"rev": 1,"signature": "SOC TEST1"},"payload_printable": "","ip_src_addr": "10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","ip_src_port": 43844,"ip_dst_addr": "10.0.0.${random():mod(254):plus(1)}","host":"nifi.soctools.geant.org","host_domain":"geant.org"},
+{"timestamp":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","alert":{"action":"allowed","category":"Potentially Bad Traffic","gid":1,"metadata":{"affected_product":["Any"],"attack_target":["Client_Endpoint"],"created_at":["2019_07_26"],"deployment":["Perimeter"],"former_category":["DNS"],"signature_severity":["Minor"],"updated_at":["2019_09_28"]},"rev":3,"severity":2,"signature":"ET DNS Query for .cc TLD","signature_id":2027758},"app_proto":"dns","destination":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":53},"dns":{"query":[{"id":37261,"rrname":"static.arduino.cc","rrtype":"A","tx_id":2,"type":"query"}]},"event_type":"alert","flow":{"bytes_toclient":1039,"bytes_toserver":343,"pkts_toclient":2,"pkts_toserver":3,"start":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}"},"flow_id":1889254052511234,"in_iface":"if1","payload":"kY0BAAABAAAAAAABBnN0YXRpYwdhcmR1aW5vAmNjAAABAAEAACkPoAAAgAAAAA==","payload_printable":".............static.arduino.cc.......)........","proto":"UDP","source":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":64164},"stream":0,"tx_id":2},
+{"timestamp":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","TLP":"AMBER","alert":{"action":"allowed","category":"Attempted Information Leak","gid":1,"metadata":{"created_at":["2014_10_15"],"former_category":["CURRENT_EVENTS"],"updated_at":["2014_10_15"]},"rev":6,"severity":2,"signature":"ET EXPLOIT SSL excessive fatal alerts (possible POODLE attack against server)","signature_id":2019418},"app_proto":"tls","destination":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":37220},"event_type":"alert","flow":{"bytes_toclient":247,"bytes_toserver":298,"pkts_toclient":4,"pkts_toserver":4,"start":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}"},"flow_id":43047386649621,"payload":"FQMAAAICKA==","payload_printable":"......(","proto":"TCP","source":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":443},"stream":1,"tls":{"ja3":{},"version":"SSLv3"},"tx_id":0}]
+</value>
+ </property>
+ <property>
+ <name>character-set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>mime.type</name>
+ <value>application/json</value>
+ </property>
+ <property>
+ <name>enrich_domain1</name>
+ <value>/host_domain</value>
+ </property>
+ <property>
+ <name>enrich_ip1</name>
+ <value>/source/ip</value>
+ </property>
+ <property>
+ <name>enrich_fqdn1</name>
+ <value>/host</value>
+ </property>
+ <property>
+ <name>data_id</name>
+ <value>suricata_alert</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-suricata-alert</value>
+ </property>
+ <property>
+ <name>data_type</name>
+ <value>suricata</value>
+ </property>
+ <property>
+ <name>enrich_ip2</name>
+ <value>/destination/ip</value>
+ </property>
+ </processor>
+ <processor>
+ <id>f8143c19-b547-1c84-90b7-2e3c37a659e7</id>
+ <name>TLS</name>
+ <position x="496.0" y="536.0" />
+ <styles />
<comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>8d37fe91-0175-1000-ffff-ffffb5c4de34</id>
- <name>Add rrname_domain++</name>
- <position x="1056.0" y="568.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/dns/rrname_domain</name>
- <value>${field.value:substringBeforeLast('.'):substringAfterLast('.'):append(${field.value:substringAfterLast('.'):prepend('.')})}</value>
- </property>
- <property>
- <name>/dns/rrname_length</name>
- <value>${field.value:length():toNumber()}</value>
- </property>
- <property>
- <name>/dns/rrname_domain_length</name>
- <value>${field.value:substringBeforeLast('.'):substringAfterLast('.'):append(${field.value:substringAfterLast('.'):prepend('.')}):length():toNumber()}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>8d312ef9-0175-1000-ffff-fffff23bbb0c</id>
- <name>Route on DNS type</name>
- <position x="1056.0" y="128.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Routing Strategy</name>
- <value>Route to Property name</value>
- </property>
- <property>
- <name>answer</name>
- <value>${type:contains("answer")}</value>
- </property>
- </processor>
- <processor>
- <id>8d2262f6-0175-1000-0000-000029eaa6ef</id>
- <name>Partition on dns message type</name>
- <position x="432.0" y="136.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.PartitionRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>type</name>
- <value>/dns/type</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>8d36474f-0175-1000-0000-00003a8dd2d0</id>
- <name>UpdateAttribute</name>
- <position x="1056.0" y="768.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>enrich_domain1</name>
- <value>/dns/rrname_domain</value>
- </property>
- <property>
- <name>enrich_fqdn1</name>
- <value>/dns/rrname</value>
- </property>
- </processor>
- <processor>
- <id>8d34409e-0175-1000-ffff-ffff99eb371d</id>
- <name>Extract rrname_domain++</name>
- <position x="1056.0" y="368.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/dns/rrname_domain</name>
- <value>/dns/rrname</value>
- </property>
- <property>
- <name>/dns/rrname_length</name>
- <value>/dns/rrname</value>
- </property>
- <property>
- <name>/dns/rrname_domain_length</name>
- <value>/dns/rrname</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>8d212c22-0175-1000-ffff-fffffbc39157</id>
- <name>Input</name>
- <position x="488.0" y="0.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>8d211b58-0175-1000-0000-000003eb5f3b</id>
- <name>Output</name>
- <position x="448.0" y="808.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>8d3979b7-0175-1000-ffff-ffffe2efe898</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d37fe91-0175-1000-ffff-ffffb5c4de34</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d36474f-0175-1000-0000-00003a8dd2d0</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d3afc9a-0175-1000-ffff-ffffe1ef144c</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d36474f-0175-1000-0000-00003a8dd2d0</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d211b58-0175-1000-0000-000003eb5f3b</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d30f240-0175-1000-ffff-ffffa4cc8a58</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d212c22-0175-1000-ffff-fffffbc39157</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>8d2262f6-0175-1000-0000-000029eaa6ef</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d3b1d93-0175-1000-ffff-ffffe953d6b9</id>
- <name />
- <bendPoints>
- <bendPoint x="568.0" y="400.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d312ef9-0175-1000-ffff-fffff23bbb0c</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d211b58-0175-1000-0000-000003eb5f3b</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>unmatched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d3821ce-0175-1000-0000-000046a72d11</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d34409e-0175-1000-ffff-ffff99eb371d</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d37fe91-0175-1000-ffff-ffffb5c4de34</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d3281c3-0175-1000-ffff-ffffed50fa50</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d2262f6-0175-1000-0000-000029eaa6ef</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d312ef9-0175-1000-ffff-fffff23bbb0c</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d3485f4-0175-1000-0000-0000175959ff</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d312ef9-0175-1000-ffff-fffff23bbb0c</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d34409e-0175-1000-ffff-ffff99eb371d</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>answer</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <connection>
- <id>349b32bb-a821-1197-ffff-ffff81dc7ff2</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d1bef35-0175-1000-0000-0000746fa33d</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>92795a59-0175-1000-ffff-ffff89bc5f21</destinationId>
- <destinationGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>tls</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d19c8d7-0175-1000-ffff-ffffe3aa385d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>1a038948-9e9a-3523-b899-990077bfd575</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>46cdd7aa-91f0-307c-90aa-65747e558f25</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d13df9c-0175-1000-0000-0000562b802e</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d13c952-0175-1000-0000-00007e8f4cae</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>24e1d8ed-10f4-3b46-958c-f2fb676e3192</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>f9a8aee6-502f-3eb9-8806-8964276d4ca0</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>24e1d8ed-10f4-3b46-958c-f2fb676e3192</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>1a038948-9e9a-3523-b899-990077bfd575</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d2364b0-0175-1000-ffff-ffffa2a4601f</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d1bef35-0175-1000-0000-0000746fa33d</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d212c22-0175-1000-ffff-fffffbc39157</destinationId>
- <destinationGroupId>8d1ad21f-0175-1000-0000-00003c540411</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>dns</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d1a6818-0175-1000-ffff-ffffeebd7e98</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>46cdd7aa-91f0-307c-90aa-65747e558f25</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>fd6b6513-51f8-3a96-a764-13bd39ec7f84</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b32e1-a821-1197-0000-00000d7cca30</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>349b32d8-a821-1197-0000-000025a75a3b</sourceId>
- <sourceGroupId>8d1afcd0-0175-1000-ffff-ffffb3690a74</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>055308a4-d020-39a9-9da4-b165796ef717</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d1c1701-0175-1000-ffff-fffff7364622</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>fd6b6513-51f8-3a96-a764-13bd39ec7f84</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d1bef35-0175-1000-0000-0000746fa33d</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>9266feff-0175-1000-ffff-ffff8c7d68c1</id>
- <name />
- <bendPoints>
- <bendPoint x="-1208.0" y="952.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d1bef35-0175-1000-0000-0000746fa33d</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>055308a4-d020-39a9-9da4-b165796ef717</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>unmatched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>9266e0c5-0175-1000-0000-00006aafc0f8</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8d211b58-0175-1000-0000-000003eb5f3b</sourceId>
- <sourceGroupId>8d1ad21f-0175-1000-0000-00003c540411</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>055308a4-d020-39a9-9da4-b165796ef717</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</id>
- <name>Mysql</name>
- <position x="-440.0" y="1272.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>14453e90-7646-1485-ffff-ffff81f3c683</id>
- <name>Add header</name>
- <position x="344.0" y="-8.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ReplaceText</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Regular Expression</name>
- <value>(?s)(^.*$)</value>
- </property>
- <property>
- <name>Replacement Value</name>
- <value>timestamp,serverhost,username,host,connectionid,queryid,operation,database,object,retcode
-</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>Maximum Buffer Size</name>
- <value>1 MB</value>
- </property>
- <property>
- <name>Replacement Strategy</name>
- <value>Prepend</value>
- </property>
- <property>
- <name>Evaluation Mode</name>
- <value>Entire text</value>
- </property>
- <property>
- <name>Line-by-Line Evaluation Mode</name>
- <value>All</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>e0bd3907-2d13-1407-b2dd-48591e65e59d</id>
- <name>UpdateRecord</name>
- <position x="-336.0" y="416.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/event_type</name>
- <value>log</value>
- </property>
- <property>
- <name>/labels/source_host</name>
- <value>${source_host}</value>
- </property>
- <property>
- <name>/timestamp</name>
- <value>${field.value:toDate('yyMMdd HH:mm:ss'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>50813f6b-a5f6-1a98-8ae4-115134714332</id>
- <name>UpdateRecord</name>
- <position x="352.0" y="472.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/event_type</name>
- <value>audit</value>
- </property>
- <property>
- <name>/labels/source_host</name>
- <value>${source_host}</value>
- </property>
- <property>
- <name>/timestamp</name>
- <value>${field.value:toDate('yyyyMMdd HH:mm:ss'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>e4353681-23e9-15af-0000-000032ea35e3</id>
- <name>RouteOnAttribute</name>
- <position x="-352.0" y="0.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Routing Strategy</name>
- <value>Route to Property name</value>
- </property>
- <property>
- <name>audit</name>
- <value>${source_file:contains("audit")}</value>
- </property>
- </processor>
- <processor>
- <id>f92d3f77-958a-1344-bd3b-7c93457e5c12</id>
- <name>Extract message</name>
- <position x="-360.0" y="-216.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>bc8e5957-0175-1000-0000-00003346421d</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>true</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>92693a34-99da-1004-adfb-bdf4aa7e1c30</id>
- <name>Convert to json</name>
- <position x="352.0" y="240.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>14453a95-7646-1485-0000-00002c675762</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>false</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>48723b8e-fae0-14e6-afdc-85c239646dc0</id>
- <name>UpdateAttribute</name>
- <position x="-320.0" y="648.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-mysql</value>
- </property>
- <property>
- <name>enrich_ip1</name>
- <value>/client.ip</value>
- </property>
- </processor>
- <processor>
- <id>14453a41-7646-1485-b398-28f819de4a45</id>
- <name>Convert to json</name>
- <position x="-336.0" y="200.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>70ea12d7-0176-1000-ffff-ffffee2ee306</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>false</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>7f683020-779c-1bc9-85da-5bad079d5d9d</id>
- <name>Input</name>
- <position x="-312.0" y="-336.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>bcbb33ba-112e-1f53-8982-d5ae9f0e701f</id>
- <name>Output</name>
- <position x="-256.0" y="960.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>14453eaa-7646-1485-0000-000070b97065</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>14453e90-7646-1485-ffff-ffff81f3c683</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>92693a34-99da-1004-adfb-bdf4aa7e1c30</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>e43535a1-23e9-15af-9f98-2061dd6f97d6</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>92693a34-99da-1004-adfb-bdf4aa7e1c30</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>50813f6b-a5f6-1a98-8ae4-115134714332</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>70e77065-0176-1000-0000-00001479fdf4</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e0bd3907-2d13-1407-b2dd-48591e65e59d</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>48723b8e-fae0-14e6-afdc-85c239646dc0</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>cf95350a-de6c-1a4b-8183-8f9cfa11449a</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>7f683020-779c-1bc9-85da-5bad079d5d9d</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>f92d3f77-958a-1344-bd3b-7c93457e5c12</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>14453fcf-7646-1485-ffff-ffff952df142</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e4353681-23e9-15af-0000-000032ea35e3</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>14453e90-7646-1485-ffff-ffff81f3c683</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>audit</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>3e21311d-dc5c-143f-b39e-d8fb8c9fd36d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>50813f6b-a5f6-1a98-8ae4-115134714332</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>48723b8e-fae0-14e6-afdc-85c239646dc0</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>14453a4b-7646-1485-ffff-fffffc8f5285</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e4353681-23e9-15af-0000-000032ea35e3</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>14453a41-7646-1485-b398-28f819de4a45</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>unmatched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>7fe931b3-82b3-1699-b49a-d380dd14a5b8</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>48723b8e-fae0-14e6-afdc-85c239646dc0</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bcbb33ba-112e-1f53-8982-d5ae9f0e701f</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>a35e3744-5906-1ee9-abc4-205356ca01d1</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>f92d3f77-958a-1344-bd3b-7c93457e5c12</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>e4353681-23e9-15af-0000-000032ea35e3</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>70e8f3cb-0176-1000-0000-00006d2cdbf5</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>14453a41-7646-1485-b398-28f819de4a45</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>e0bd3907-2d13-1407-b2dd-48591e65e59d</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>5d04357e-423c-1ab5-a7a4-44565abfed7f</id>
- <name>Haproxy</name>
- <position x="-448.0" y="664.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>c9763c4c-7186-1460-871a-b5fd00ca3241</id>
- <name>UpdateRecord</name>
- <position x="352.0" y="472.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/labels/source_host</name>
- <value>${source_host}</value>
- </property>
- <property>
- <name>/timestamp</name>
- <value>${field.value:toDate('dd/MMM/yyyy:HH:mm:ss.SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>e4c8356d-54ad-15b5-94fe-799d9465aa51</id>
- <name>Extract message</name>
- <position x="352.0" y="280.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>bc8e5957-0175-1000-0000-00003346421d</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>true</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</id>
- <name>Convert to json</name>
- <position x="1064.0" y="272.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>56ebe0aa-0176-1000-ffff-ffffbd212f01</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>false</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</id>
- <name>UpdateAttribute</name>
- <position x="1072.0" y="472.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-haproxy</value>
- </property>
- <property>
- <name>enrich_ip1</name>
- <value>/client.ip</value>
- </property>
- </processor>
- <inputPort>
- <id>65a33e05-e157-1bfc-8741-adf11b3df720</id>
- <name>Input</name>
- <position x="397.9999517774115" y="110.99999315685733" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>328b35e2-eb52-1f47-b84d-52941eff8a07</id>
- <name>Output</name>
- <position x="1120.0" y="808.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>960f3ac9-95dc-103d-a70a-ca3b070851a4</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>328b35e2-eb52-1f47-b84d-52941eff8a07</destinationId>
- <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>0ecb3e12-768e-1896-a850-2a2bec52eb95</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>c9763c4c-7186-1460-871a-b5fd00ca3241</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>7fbd38e8-60a2-1503-8a6c-ffc6b156b3b0</destinationId>
- <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>b5d43cea-5555-10b0-b75f-b88a95e9c6aa</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>65a33e05-e157-1bfc-8741-adf11b3df720</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>e4c8356d-54ad-15b5-94fe-799d9465aa51</destinationId>
- <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>484a3eab-4af3-11cd-abe2-d5ee6fc1a291</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e4c8356d-54ad-15b5-94fe-799d9465aa51</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</destinationId>
- <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>39ef3a2d-874e-11a6-87be-0b3582fa43de</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>f6e63fd3-6150-1d72-a58a-46b43bc5d5c2</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c9763c4c-7186-1460-871a-b5fd00ca3241</destinationId>
- <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>7263390f-914c-1f6e-9451-75f908ed8816</id>
- <name>Elasticsearch</name>
- <position x="-1904.0" y="488.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>295133bd-42e6-1b08-80c5-bea2e19921fc</id>
- <name>UpdateAttribute</name>
- <position x="360.0" y="600.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-elasticsearch</value>
- </property>
- </processor>
- <inputPort>
- <id>39ce3238-1ebd-1c2c-b724-01d18f147b6f</id>
- <name>Input</name>
- <position x="408.0" y="320.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>bbc63756-9681-13b9-8c07-20c82f62ceca</id>
- <name>Output</name>
- <position x="408.0" y="920.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>15e0341e-6dd3-172a-b2b5-8f1d5740fea1</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>39ce3238-1ebd-1c2c-b724-01d18f147b6f</sourceId>
- <sourceGroupId>7263390f-914c-1f6e-9451-75f908ed8816</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>295133bd-42e6-1b08-80c5-bea2e19921fc</destinationId>
- <destinationGroupId>7263390f-914c-1f6e-9451-75f908ed8816</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>af99379e-bf26-19c5-bd70-bd6d405fb0b7</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>295133bd-42e6-1b08-80c5-bea2e19921fc</sourceId>
- <sourceGroupId>7263390f-914c-1f6e-9451-75f908ed8816</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bbc63756-9681-13b9-8c07-20c82f62ceca</destinationId>
- <destinationGroupId>7263390f-914c-1f6e-9451-75f908ed8816</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</id>
- <name>Keycloak</name>
- <position x="-440.0" y="1064.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>8e17350e-583e-1130-8ec7-bd2dc5d4f361</id>
- <name>UpdateAttribute</name>
- <position x="344.0" y="736.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-keycloak</value>
- </property>
- </processor>
- <processor>
- <id>fbbe3f9c-5336-11c9-0000-00003ab5dde5</id>
- <name>Fix timestamp</name>
- <position x="352.0" y="480.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.jolt.record.JoltTransformRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-jolt-record-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>jolt-record-record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>jolt-record-record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>jolt-record-transform</name>
- <value>jolt-transform-chain</value>
- </property>
- <property>
- <name>jolt-record-custom-class</name>
- </property>
- <property>
- <name>jolt-record-custom-modules</name>
- </property>
- <property>
- <name>jolt-record-spec</name>
- <value>[
- {
- "operation": "shift",
- "spec": {
- "timestamp": {
- "1": "timestamp"
- },
- "*": "&"
- }
- }
-]</value>
- </property>
- <property>
- <name>jolt-record-transform-cache-size</name>
- <value>1</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>10cb3b64-e867-1d81-bd59-eb9cf6883f24</id>
- <name>Input</name>
- <position x="408.0" y="320.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>84dc3511-1322-175b-8083-9729037f8edb</id>
- <name>Output</name>
- <position x="392.0" y="984.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>fbbe3fbf-5336-11c9-ffff-ffffb7c3576e</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>fbbe3f9c-5336-11c9-0000-00003ab5dde5</sourceId>
- <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8e17350e-583e-1130-8ec7-bd2dc5d4f361</destinationId>
- <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>50c83129-28e1-1d45-bafe-912df3cdf284</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>10cb3b64-e867-1d81-bd59-eb9cf6883f24</sourceId>
- <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>fbbe3f9c-5336-11c9-0000-00003ab5dde5</destinationId>
- <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>fbbe3ede-5336-11c9-8870-deb7fffd14ae</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8e17350e-583e-1130-8ec7-bd2dc5d4f361</sourceId>
- <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>84dc3511-1322-175b-8083-9729037f8edb</destinationId>
- <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>83691174-683f-3c7c-8526-8fc00397aee1</id>
- <name>Zeek</name>
- <position x="-448.0" y="464.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>fec43039-de5d-1e3b-850a-5e25d7b93c76</id>
- <name>UpdateAttribute</name>
- <position x="-1121.584644408096" y="913.629598069974" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-uninett-darknet</value>
- </property>
- <property>
- <name>mime.type</name>
- <value>application/json</value>
- </property>
- <property>
- <name>enrich_ip2</name>
- <value>/destination/ip</value>
- </property>
- <property>
- <name>enrich_ip1</name>
- <value>/source/ip</value>
- </property>
- <property>
- <name>TLP</name>
- <value>AMBER</value>
- </property>
- </processor>
- <processor>
- <id>9027e415-c8cd-355e-af16-0c635f43832f</id>
- <name>Convert timestamp and add fields</name>
- <position x="-472.77987807459795" y="696.7530680701591" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/data_type</name>
- <value>zeek</value>
- </property>
- <property>
- <name>/data_index</name>
- <value>logs-zeek-conn</value>
- </property>
- <property>
- <name>/data_id</name>
- <value>zeek_conn</value>
- </property>
- <property>
- <name>/timestamp</name>
- <value>${field.value:multiply(1000):format('yyyy-MM-dd HH:mm:ss.SSSZ'):replace(' ','T'):replaceFirst('\+(\d\d)(\d\d)','+$1:$2')}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</id>
- <name>Normalize fields</name>
- <position x="-1121.1565561587029" y="706.9002449806696" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.jolt.record.JoltTransformRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-jolt-record-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>jolt-record-record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>jolt-record-record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>jolt-record-transform</name>
- <value>jolt-transform-chain</value>
- </property>
- <property>
- <name>jolt-record-custom-class</name>
- </property>
- <property>
- <name>jolt-record-custom-modules</name>
- </property>
- <property>
- <name>jolt-record-spec</name>
- <value>[{
- "operation": "modify-overwrite-beta",
- "spec": {
- "*": "=recursivelySquashNulls"
- }
- },{
- "operation": "shift",
- "spec": {
- "*": {
- "json": {
- "*": "[&2].&"
- },
- "host": {
- "name": "[&2].labels.source_host"
- },
- "source": "[&1].labels.source"
- }
- }
-}, {
- "operation": "shift",
- "spec": {
- "*": {
- "ts": "[&1].timestamp",
- "id.resp_h":"[&1].destination.ip",
- "id.resp_p":"[&1].destination.port",
- "id.orig_h":"[&1].source.ip",
- "id.orig_p":"[&1].source.port",
- "*": "[&1].&"
- }
- }
-}, {
- "operation": "modify-overwrite-beta",
- "spec": {
- "*": {
- "timestamp": "=toString"
- }
- }
-}]</value>
- </property>
- <property>
- <name>jolt-record-transform-cache-size</name>
- <value>1</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>349b3362-a821-1197-ffff-ffff91d0e6c0</id>
- <name>Input</name>
- <position x="-1072.0" y="520.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>a28a9e95-1003-3ea6-9af6-a334c1aec07c</id>
- <name>To enrichment</name>
- <position x="-1065.7090714972117" y="1164.8389289189608" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>c8be8213-0170-1000-0000-0000695bc36c</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>fec43039-de5d-1e3b-850a-5e25d7b93c76</sourceId>
- <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</destinationId>
- <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>2d0f222e-d08e-31fd-b5e1-1ce178368e4c</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</sourceId>
- <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>9027e415-c8cd-355e-af16-0c635f43832f</destinationId>
- <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>c8be6e60-0170-1000-ffff-ffffe34d52ef</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>9027e415-c8cd-355e-af16-0c635f43832f</sourceId>
- <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>fec43039-de5d-1e3b-850a-5e25d7b93c76</destinationId>
- <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b3364-a821-1197-0000-000063d0c208</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>349b3362-a821-1197-ffff-ffff91d0e6c0</sourceId>
- <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>3bf497f2-3aed-3465-b91c-72ef6e53f0ea</destinationId>
- <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>31b13b40-8e26-1798-9777-2272881c6031</id>
- <name>Zookeeper</name>
- <position x="-440.0" y="1488.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>b09b367b-060f-1e74-9a96-ca5ba5f88858</id>
- <name>UpdateRecord</name>
- <position x="352.0" y="472.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UpdateRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>replacement-value-strategy</name>
- <value>literal-value</value>
- </property>
- <property>
- <name>/labels/source_host</name>
- <value>${source_host}</value>
- </property>
- <property>
- <name>/timestamp</name>
- <value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>dd3b361c-4e9c-158c-ba31-61006a0b21b7</id>
- <name>UpdateAttribute</name>
- <position x="1072.0" y="472.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-zookeeper</value>
- </property>
- </processor>
- <processor>
- <id>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</id>
- <name>Extract message</name>
- <position x="352.0" y="280.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>bc8e5957-0175-1000-0000-00003346421d</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>true</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>38f03e95-db26-1287-be1a-4218f647596a</id>
- <name>Convert to json</name>
- <position x="1064.0" y="272.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ConvertRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>record-reader</name>
- <value>7504a565-0176-1000-ffff-ffff9c0f0741</value>
- </property>
- <property>
- <name>record-writer</name>
- <value>17b30955-5464-3709-8a32-69a459850cfa</value>
- </property>
- <property>
- <name>include-zero-record-flowfiles</name>
- <value>false</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>a77d3c33-9575-1926-b230-9cf1fca55e7e</id>
- <name>Input</name>
- <position x="397.9999517774115" y="110.99999315685733" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>f1e73393-065a-1450-9ad0-fd7cdb57853f</id>
- <name>Output</name>
- <position x="1120.0" y="808.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>4b353d86-8a69-1ca1-bc1c-8db0049f5886</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>38f03e95-db26-1287-be1a-4218f647596a</sourceId>
- <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>b09b367b-060f-1e74-9a96-ca5ba5f88858</destinationId>
- <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>b05339e8-7bbc-1975-a9b2-4bc789dda2df</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>a77d3c33-9575-1926-b230-9cf1fca55e7e</sourceId>
- <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</destinationId>
- <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>5d6f3995-2075-11db-ba4b-1b76e1ed6473</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>b09b367b-060f-1e74-9a96-ca5ba5f88858</sourceId>
- <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>dd3b361c-4e9c-158c-ba31-61006a0b21b7</destinationId>
- <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>2a0131b7-9c0d-157d-a9ac-abd12398f2a8</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>4b1c38b7-8f98-1a81-96c9-17e3eccc45b2</sourceId>
- <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>38f03e95-db26-1287-be1a-4218f647596a</destinationId>
- <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>5a2e3db4-49d3-187d-9db9-3f0b48a2c6af</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>dd3b361c-4e9c-158c-ba31-61006a0b21b7</sourceId>
- <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>f1e73393-065a-1450-9ad0-fd7cdb57853f</destinationId>
- <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
- <name>Kibana</name>
- <position x="-440.0" y="864.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>992c3710-1c87-169c-ab17-d2597387a25e</id>
- <name>UpdateAttribute</name>
- <position x="360.0" y="512.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-kibana</value>
- </property>
- </processor>
- <inputPort>
- <id>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</id>
- <name>Input</name>
- <position x="408.0" y="320.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <outputPort>
- <id>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</id>
- <name>Output</name>
- <position x="408.0" y="760.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>cc403fb4-8d68-1c68-82c3-b9af4affddaa</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</sourceId>
- <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>992c3710-1c87-169c-ab17-d2597387a25e</destinationId>
- <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>b9e33c29-910f-134a-8390-2970800d7fcf</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>992c3710-1c87-169c-ab17-d2597387a25e</sourceId>
- <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</destinationId>
- <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <connection>
- <id>fbbe3f1b-5336-11c9-ffff-ffffd29d2f5c</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>84dc3511-1322-175b-8083-9729037f8edb</sourceId>
- <sourceGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>56e5f029-0176-1000-ffff-fffff7512a3b</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>328b35e2-eb52-1f47-b84d-52941eff8a07</sourceId>
- <sourceGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>e43535d0-23e9-15af-ffff-ffffa44d6172</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bcbb33ba-112e-1f53-8982-d5ae9f0e701f</sourceId>
- <sourceGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b339b-a821-1197-0000-00002e648df6</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>a28a9e95-1003-3ea6-9af6-a334c1aec07c</sourceId>
- <sourceGroupId>83691174-683f-3c7c-8526-8fc00397aee1</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>75109cc1-0176-1000-ffff-ffff86db235d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>f1e73393-065a-1450-9ad0-fd7cdb57853f</sourceId>
- <sourceGroupId>31b13b40-8e26-1798-9777-2272881c6031</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d0ea3d4-0175-1000-0000-0000471b8522</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>89639d3d-0175-1000-ffff-ffffb446c257</destinationId>
- <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>unmatched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>7558e6dd-0176-1000-ffff-ffffec9061a8</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>39ce3238-1ebd-1c2c-b724-01d18f147b6f</destinationId>
- <destinationGroupId>7263390f-914c-1f6e-9451-75f908ed8816</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>elasticsearch</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>fbbe3ee1-5336-11c9-ffff-ffffa7c97811</id>
- <name />
- <bendPoints>
- <bendPoint x="-720.0" y="1016.0" />
- <bendPoint x="-584.0" y="1152.0" />
- </bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>10cb3b64-e867-1d81-bd59-eb9cf6883f24</destinationId>
- <destinationGroupId>f88732b0-d93f-1f6e-ba01-40b41ea20fe3</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>keycloak</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8d1fe825-0175-1000-ffff-fffff0505cdc</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8d13c952-0175-1000-0000-00007e8f4cae</destinationId>
- <destinationGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>suricata</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>74ff448b-0176-1000-0000-00002e302e83</id>
- <name />
- <bendPoints>
- <bendPoint x="-688.0" y="1576.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>a77d3c33-9575-1926-b230-9cf1fca55e7e</destinationId>
- <destinationGroupId>31b13b40-8e26-1798-9777-2272881c6031</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>zookeeper</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b3398-a821-1197-ffff-ffffc5ae6471</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>349b3362-a821-1197-ffff-ffff91d0e6c0</destinationId>
- <destinationGroupId>83691174-683f-3c7c-8526-8fc00397aee1</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>zeek</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>61c51cd8-0176-1000-ffff-ffff9247ba7c</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>887c36a6-39d6-1b60-8a83-d4d10ea7e03b</sourceId>
- <sourceGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bc6e50cc-0175-1000-ffff-ffffbd982e0c</id>
- <name />
- <bendPoints />
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>bc6c2159-0175-1000-ffff-ffffb4de4d47</destinationId>
- <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>nifi</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>56e5add3-0176-1000-ffff-ffffd667d1f6</id>
- <name />
- <bendPoints>
- <bendPoint x="-584.0" y="624.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>65a33e05-e157-1bfc-8741-adf11b3df720</destinationId>
- <destinationGroupId>5d04357e-423c-1ab5-a7a4-44565abfed7f</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>haproxy</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>6196cd03-0176-1000-ffff-ffffd39b8c82</id>
- <name />
- <bendPoints>
- <bendPoint x="-576.0" y="896.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</destinationId>
- <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>kibana</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>89630460-0175-1000-0000-00006b5f18c8</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>89620b1a-0175-1000-0000-000078566f34</sourceId>
- <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>8962ad5a-0175-1000-ffff-ffffde6db5a6</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b3303-a821-1197-ffff-ffffa12b866d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8963b202-0175-1000-0000-000022d64ba2</sourceId>
- <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bcadaf87-0175-1000-0000-000048464ec3</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bca9636a-0175-1000-0000-000013fa95aa</sourceId>
- <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>bcb879d5-0175-1000-0000-000070879ad0</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b3301-a821-1197-0000-0000070259c4</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>055308a4-d020-39a9-9da4-b165796ef717</sourceId>
- <sourceGroupId>bd12dc14-015e-3428-bfdf-b1219d2d6fdb</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>349b32fe-a821-1197-0000-00003a0b6fe5</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>e43535c9-23e9-15af-ffff-ffffcd7d888a</id>
- <name />
- <bendPoints>
- <bendPoint x="-704.0" y="1256.0" />
- <bendPoint x="-584.0" y="1368.0" />
- </bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>7f683020-779c-1bc9-85da-5bad079d5d9d</destinationId>
- <destinationGroupId>48bc31b5-dbc5-116d-adbe-fe0f10314ac2</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship>mysql</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>7575486d-0176-1000-0000-00002542d6de</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bbc63756-9681-13b9-8c07-20c82f62ceca</sourceId>
- <sourceGroupId>7263390f-914c-1f6e-9451-75f908ed8816</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>bcb879d5-0175-1000-0000-000070879ad0</destinationId>
- <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>b3d57504-7c06-37a3-b59b-8723f60fa728</id>
- <name>Test data</name>
- <position x="-496.0" y="552.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <outputPort>
- <id>d30dc946-251a-307c-8e88-f2262b0bb194</id>
- <name>To enrichment</name>
- <position x="731.0454088698874" y="433.2315817172085" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <processGroup>
- <id>0c83ef26-0175-1000-ffff-ffffcac37910</id>
- <name>Suricata</name>
- <position x="462.0553417896858" y="119.99261716112323" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>bb7dc9ff-2d25-3134-9617-cca3cabe9179</id>
- <name>Alerts</name>
- <position x="496.0" y="392.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>10 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>File Size</name>
- <value>0B</value>
- </property>
- <property>
- <name>Batch Size</name>
- <value>1</value>
- </property>
- <property>
- <name>Data Format</name>
- <value>Text</value>
- </property>
- <property>
- <name>Unique FlowFiles</name>
- <value>false</value>
- </property>
- <property>
- <name>generate-ff-custom-text</name>
- <value>[{"stream": 0,"flow": {"bytes_toserver": 74,"bytes_toclient": 0,"start": "${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","pkts_toserver": 1,"pkts_toclient": 0},"vlan": 665,"ip_dst_port": 54323,"in_iface": "ens1f3","payload": "","timestamp": "${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","proto": "TCP","event_type": "alert","alert": {"category": "Not Suspicious Traffic","severity": 3,"action": "allowed","gid": 1,"signature_id": 29999991,"rev": 1,"signature": "SOC TEST1"},"payload_printable": "","ip_src_addr": "10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","ip_src_port": 43844,"ip_dst_addr": "10.0.0.${random():mod(254):plus(1)}","host":"nifi.soctools.geant.org","host_domain":"geant.org"},
-{"timestamp":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","alert":{"action":"allowed","category":"Potentially Bad Traffic","gid":1,"metadata":{"affected_product":["Any"],"attack_target":["Client_Endpoint"],"created_at":["2019_07_26"],"deployment":["Perimeter"],"former_category":["DNS"],"signature_severity":["Minor"],"updated_at":["2019_09_28"]},"rev":3,"severity":2,"signature":"ET DNS Query for .cc TLD","signature_id":2027758},"app_proto":"dns","destination":{"ip":"10.10.10.${random():mod(254):plus(1)}","port":53},"dns":{"query":[{"id":37261,"rrname":"example.evil","rrtype":"A","tx_id":2,"type":"query"}]},"event_type":"alert","flow":{"bytes_toclient":1039,"bytes_toserver":343,"pkts_toclient":2,"pkts_toserver":3,"start":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}"},"flow_id":1889254052511234,"in_iface":"if1","payload":"kY0BAAABAAAAAAABBnN0YXRpYwdhcmR1aW5vAmNjAAABAAEAACkPoAAAgAAAAA==","payload_printable":".............example.evil.......)........","proto":"UDP","source":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":64164},"stream":0,"tx_id":2},
-{"timestamp":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","TLP":"AMBER","alert":{"action":"allowed","category":"Attempted Information Leak","gid":1,"metadata":{"created_at":["2014_10_15"],"former_category":["CURRENT_EVENTS"],"updated_at":["2014_10_15"]},"rev":6,"severity":2,"signature":"ET EXPLOIT SSL excessive fatal alerts (possible POODLE attack against server)","signature_id":2019418},"app_proto":"tls","destination":{"ip":"10.10.10.${random():mod(10):plus(1)}","port":37220},"event_type":"alert","flow":{"bytes_toclient":247,"bytes_toserver":298,"pkts_toclient":4,"pkts_toserver":4,"start":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}"},"flow_id":43047386649621,"payload":"FQMAAAICKA==","payload_printable":"......(","proto":"TCP","source":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":443},"stream":1,"tls":{"ja3":{},"version":"SSLv3"},"tx_id":0}]</value>
- </property>
- <property>
- <name>character-set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>mime-type</name>
- </property>
- <property>
- <name>mime.type</name>
- <value>application/json</value>
- </property>
- <property>
- <name>enrich_domain1</name>
- <value>/host_domain</value>
- </property>
- <property>
- <name>enrich_ip1</name>
- <value>/source/ip</value>
- </property>
- <property>
- <name>enrich_fqdn1</name>
- <value>/host</value>
- </property>
- <property>
- <name>data_id</name>
- <value>suricata_alert</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-suricata-alert</value>
- </property>
- <property>
- <name>data_type</name>
- <value>suricata</value>
- </property>
- <property>
- <name>enrich_ip2</name>
- <value>/destination/ip</value>
- </property>
- </processor>
- <processor>
- <id>f8143c19-b547-1c84-90b7-2e3c37a659e7</id>
- <name>TLS</name>
- <position x="496.0" y="536.0" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>10 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>File Size</name>
- <value>0B</value>
- </property>
- <property>
- <name>Batch Size</name>
- <value>1</value>
- </property>
- <property>
- <name>Data Format</name>
- <value>Text</value>
- </property>
- <property>
- <name>Unique FlowFiles</name>
- <value>false</value>
- </property>
- <property>
- <name>generate-ff-custom-text</name>
- <value>[{"timestamp":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","TLP":"AMBER","destination":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":443},"event_type":"tls","flow_id":852792667052212,"in_iface":"if1","proto":"TCP","source":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":53466},"tls":{"ja3":{"hash":"e5b607b5862a46cab44d7bacd582b3cd","string":"771,4867-4865-4866-52393-52392-49195-49199-49196-49200-49171-49172-156-157-47-53-10,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0"},"sni":"clients3.google.com","sni_domain":"google.com","version":"TLS 1.3"}}]</value>
- </property>
- <property>
- <name>character-set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>mime-type</name>
- </property>
- <property>
- <name>mime.type</name>
- <value>application/json</value>
- </property>
- <property>
- <name>enrich_domain1</name>
- <value>/tls/sni_domain</value>
- </property>
- <property>
- <name>enrich_fqdn1</name>
- <value>/tls/sni</value>
- </property>
- <property>
- <name>enrich_ip1</name>
- <value>/source/ip</value>
- </property>
- <property>
- <name>data_id</name>
- <value>suricata_tls</value>
- </property>
- <property>
- <name>data_index</name>
- <value>logs-suricata-tls</value>
- </property>
- <property>
- <name>data_type</name>
- <value>suricata</value>
- </property>
- <property>
- <name>enrich_ip2</name>
- <value>/destination/ip</value>
- </property>
- </processor>
- <outputPort>
- <id>0c864b15-0175-1000-0000-00001d403b1e</id>
- <name>To enrichment</name>
- <position x="1192.0" y="576.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </outputPort>
- <connection>
- <id>1cb3658c-0175-1000-ffff-ffff93193081</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>f8143c19-b547-1c84-90b7-2e3c37a659e7</sourceId>
- <sourceGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>0c864b15-0175-1000-0000-00001d403b1e</destinationId>
- <destinationGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>0c865d01-0175-1000-0000-0000559b408d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bb7dc9ff-2d25-3134-9617-cca3cabe9179</sourceId>
- <sourceGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>0c864b15-0175-1000-0000-00001d403b1e</destinationId>
- <destinationGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <connection>
- <id>0c866fa6-0175-1000-ffff-ffffe866c936</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>0c864b15-0175-1000-0000-00001d403b1e</sourceId>
- <sourceGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>d30dc946-251a-307c-8e88-f2262b0bb194</destinationId>
- <destinationGroupId>b3d57504-7c06-37a3-b59b-8723f60fa728</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <connection>
- <id>c5fe676f-baa5-3d90-956e-fe502db0ac68</id>
- <name />
- <bendPoints />
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>d30dc946-251a-307c-8e88-f2262b0bb194</sourceId>
- <sourceGroupId>b3d57504-7c06-37a3-b59b-8723f60fa728</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>20b01ab3-3a8d-3573-b95d-a4a45494050f</destinationId>
- <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>349b33a3-a821-1197-0000-00001ce4370e</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>349b32fe-a821-1197-0000-00003a0b6fe5</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>20b01ab3-3a8d-3573-b95d-a4a45494050f</destinationId>
- <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bcb8ef9d-0175-1000-0000-000017e52ef1</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>bcb879d5-0175-1000-0000-000070879ad0</sourceId>
- <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>21a9e277-2d80-359a-9c57-cb76d8962e6d</destinationId>
- <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>27d64272-0172-1000-0000-000079e1c9c6</id>
- <name />
- <bendPoints>
- <bendPoint x="88.0" y="864.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>27d5761b-0172-1000-0000-000059275dad</sourceId>
- <sourceGroupId>27d51d04-0172-1000-0000-00004573c6ec</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>20b01ab3-3a8d-3573-b95d-a4a45494050f</destinationId>
- <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>27d65fe7-0172-1000-ffff-ffffec2db03b</id>
- <name />
- <bendPoints>
- <bendPoint x="-744.0" y="856.0" />
- </bendPoints>
- <labelIndex>0</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>27d5dab2-0172-1000-ffff-ffffab5c50be</sourceId>
- <sourceGroupId>27d51d04-0172-1000-0000-00004573c6ec</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>21a9e277-2d80-359a-9c57-cb76d8962e6d</destinationId>
- <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
- <destinationType>OUTPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <controllerService>
- <id>94600c6c-704e-3ff8-a2a4-f2f25c71dc3b</id>
- <name>JsonRecordSetWriter</name>
- <comment />
- <class>org.apache.nifi.json.JsonRecordSetWriter</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>Schema Write Strategy</name>
- <value>no-schema</value>
- </property>
- <property>
- <name>schema-cache</name>
- </property>
- <property>
- <name>schema-protocol-version</name>
- <value>1</value>
- </property>
- <property>
- <name>schema-access-strategy</name>
- <value>inherit-record-schema</value>
- </property>
- <property>
- <name>schema-registry</name>
- </property>
- <property>
- <name>schema-name</name>
- <value>${schema.name}</value>
- </property>
- <property>
- <name>schema-version</name>
- </property>
- <property>
- <name>schema-branch</name>
- </property>
- <property>
- <name>schema-text</name>
- <value>${avro.schema}</value>
- </property>
- <property>
- <name>Date Format</name>
- </property>
- <property>
- <name>Time Format</name>
- </property>
- <property>
- <name>Timestamp Format</name>
- </property>
- <property>
- <name>Pretty Print JSON</name>
- <value>false</value>
- </property>
- <property>
- <name>suppress-nulls</name>
- <value>suppress-missing</value>
- </property>
- <property>
- <name>output-grouping</name>
- <value>output-array</value>
- </property>
- <property>
- <name>compression-format</name>
- <value>none</value>
- </property>
- <property>
- <name>compression-level</name>
- <value>1</value>
- </property>
- </controllerService>
- <controllerService>
- <id>09b4fa02-0459-358d-939f-54fda8aea702</id>
- <name>VolatileSchemaCache</name>
- <comment />
- <class>org.apache.nifi.schema.inference.VolatileSchemaCache</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>false</enabled>
- <property>
- <name>max-cache-size</name>
- <value>100</value>
- </property>
- </controllerService>
- </processGroup>
- <processGroup>
- <id>e9c19adc-c8a4-327e-ad24-24e71fd3474e</id>
- <name>Data output</name>
- <position x="829.4446253936723" y="1015.2711478364996" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <inputPort>
- <id>e333b82d-7408-3747-8dd2-46473704e51b</id>
- <name>Data input</name>
- <position x="-688.0" y="496.0" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <processGroup>
- <id>7ebf304b-4978-3adc-ac31-470fb76e5029</id>
- <name>Elastic odfe</name>
- <position x="-759.1319580078125" y="739.6137390136719" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>8b48f28f-2379-3f0f-81fe-4e1b93e72666</id>
- <name>PutElasticsearchHttpRecord</name>
- <position x="-856.2311706542969" y="629.8186340332031" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.elasticsearch.PutElasticsearchHttpRecord</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-elasticsearch-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>elasticsearch-http-url</name>
- <value>${elastic_url}</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>Username</name>
- <value>${elastic_username}</value>
- </property>
- <property>
- <name>Password</name>
- <value>enc{e3c6c99d66e95dfa569c6dab15f7bd5cb2142d215044a4c556aba0a2bed19ac85c899bd8837e09bb49300f0823011b45}</value>
- </property>
- <property>
- <name>elasticsearch-http-connect-timeout</name>
- <value>5 secs</value>
- </property>
- <property>
- <name>elasticsearch-http-response-timeout</name>
- <value>15 secs</value>
- </property>
- <property>
- <name>proxy-configuration-service</name>
- </property>
- <property>
- <name>elasticsearch-http-proxy-host</name>
- </property>
- <property>
- <name>elasticsearch-http-proxy-port</name>
- </property>
- <property>
- <name>proxy-username</name>
- </property>
- <property>
- <name>proxy-password</name>
- </property>
- <property>
- <name>put-es-record-record-reader</name>
- <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
- </property>
- <property>
- <name>put-es-record-record-writer</name>
- </property>
- <property>
- <name>put-es-record-log-all-errors</name>
- <value>false</value>
- </property>
- <property>
- <name>put-es-record-id-path</name>
- </property>
- <property>
- <name>put-es-record-index</name>
- <value>${data_index}-${now():format("yyyy-MM-dd")}</value>
- </property>
- <property>
- <name>put-es-record-type</name>
- <value>_doc</value>
- </property>
- <property>
- <name>put-es-record-index-op</name>
- <value>index</value>
- </property>
- <property>
- <name>suppress-nulls</name>
- <value>always-suppress</value>
- </property>
- <property>
- <name>Date Format</name>
- </property>
- <property>
- <name>Time Format</name>
- </property>
- <property>
- <name>Timestamp Format</name>
- </property>
- <autoTerminatedRelationship>success</autoTerminatedRelationship>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <inputPort>
- <id>e7d34e01-babe-3022-ad9b-a7620e7c0f38</id>
- <name>Data input</name>
- <position x="-803.9990234375" y="484.5271301269531" />
- <comments />
- <scheduledState>RUNNING</scheduledState>
- </inputPort>
- <connection>
- <id>3280c550-2117-37a6-8b5e-3bc1953fa17e</id>
- <name />
- <bendPoints>
- <bendPoint x="-393.2311706542969" y="669.8186340332031" />
- <bendPoint x="-393.2311706542969" y="719.8186340332031" />
- </bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</sourceId>
- <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId>
- <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>retry</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>5de8f98f-ce46-3565-b0ce-7f8ecf518c53</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</sourceId>
- <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId>
- <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>60 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>2bb2f914-0172-1000-0000-0000240c76e4</id>
- <name>Custom output</name>
- <position x="-160.0" y="736.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <inputPort>
- <id>2bb31aa5-0172-1000-0000-00000869fb70</id>
- <name>Input</name>
- <position x="-648.0" y="496.0" />
- <comments />
- <scheduledState>STOPPED</scheduledState>
- </inputPort>
- </processGroup>
- <connection>
- <id>9349cb73-0175-1000-ffff-ffff90dc265d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e333b82d-7408-3747-8dd2-46473704e51b</sourceId>
- <sourceGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>2bb31aa5-0172-1000-0000-00000869fb70</destinationId>
- <destinationGroupId>2bb2f914-0172-1000-0000-0000240c76e4</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>1 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>34772170-2400-3eb6-b9c5-c03b912a38f3</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e333b82d-7408-3747-8dd2-46473704e51b</sourceId>
- <sourceGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</sourceGroupId>
- <sourceType>INPUT_PORT</sourceType>
- <destinationId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</destinationId>
- <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <connection>
- <id>875a975e-46e1-36fa-a035-4799201abd63</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>20b01ab3-3a8d-3573-b95d-a4a45494050f</sourceId>
- <sourceGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>74abf119-faa6-3e9e-bb31-da2e79f89a38</destinationId>
- <destinationGroupId>fcbcacd1-542d-3a15-a5aa-9c1302328954</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bbc37560-0171-1000-0000-000055178fff</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>c164884d-277f-31af-ac3c-18b211667bbf</sourceId>
- <sourceGroupId>fcbcacd1-542d-3a15-a5aa-9c1302328954</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>e333b82d-7408-3747-8dd2-46473704e51b</destinationId>
- <destinationGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>9cdaaee8-0e39-3dbd-a7cc-06a89056bb7c</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>21a9e277-2d80-359a-9c57-cb76d8962e6d</sourceId>
- <sourceGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</sourceGroupId>
- <sourceType>OUTPUT_PORT</sourceType>
- <destinationId>e333b82d-7408-3747-8dd2-46473704e51b</destinationId>
- <destinationGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</destinationGroupId>
- <destinationType>INPUT_PORT</destinationType>
- <relationship />
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <controllerService>
- <id>b7794eb3-9227-36dd-8751-e87d1c2321ee</id>
- <name>Misp DistributedMapCacheClientService</name>
- <comment />
- <class>org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-distributed-cache-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>Server Hostname</name>
- <value>localhost</value>
- </property>
- <property>
- <name>Server Port</name>
- <value>6000</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- </property>
- <property>
- <name>Communications Timeout</name>
- <value>30 secs</value>
- </property>
- </controllerService>
- </processGroup>
- <processGroup>
- <id>72eb009e-0c2f-302d-bc6c-2d02c29c25a9</id>
- <name>Enrichment data</name>
- <position x="1720.0" y="248.0" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processGroup>
- <id>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</id>
- <name>Top domains</name>
- <position x="970.3727876614566" y="673.4981494769316" />
- <comment>Downloads CSV files containing top domains from Alexa and Umbrella</comment>
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>857cd537-4aeb-31fb-9740-0513e6cc46fe</id>
- <name>Unzip CSV files</name>
- <position x="-297.30227379373514" y="212.70767899178307" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UnpackContent</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Packaging Format</name>
- <value>zip</value>
- </property>
- <property>
- <name>File Filter</name>
- <value>.*</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>937de5fc-7d4a-35af-a071-46f04d6ea4fa</id>
- <name>Save to disk</name>
- <position x="326.18698401876486" y="392.4228279175642" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.PutFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Directory</name>
- <value>/opt/nifi/nifi-current/conf/</value>
- </property>
- <property>
- <name>Conflict Resolution Strategy</name>
- <value>replace</value>
- </property>
- <property>
- <name>Create Missing Directories</name>
- <value>true</value>
- </property>
- <property>
- <name>Maximum File Count</name>
- </property>
- <property>
- <name>Last Modified Time</name>
- </property>
- <property>
- <name>Permissions</name>
- </property>
- <property>
- <name>Owner</name>
- </property>
- <property>
- <name>Group</name>
- </property>
- <autoTerminatedRelationship>success</autoTerminatedRelationship>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>3c4d65a9-aa39-380f-b16b-2aea028a019b</id>
- <name>Download Alexa CSV file</name>
- <position x="197.54468055196799" y="-60.57735518790443" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GetHTTP</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>1 day</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>URL</name>
- <value>http://s3.amazonaws.com/alexa-static/top-1m.csv.zip</value>
- </property>
- <property>
- <name>Filename</name>
- <value>alexa-top-1m.csv.zip</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- </property>
- <property>
- <name>Username</name>
- </property>
- <property>
- <name>Password</name>
- </property>
- <property>
- <name>Connection Timeout</name>
- <value>30 sec</value>
- </property>
- <property>
- <name>Data Timeout</name>
- <value>10 min</value>
- </property>
- <property>
- <name>User Agent</name>
- </property>
- <property>
- <name>Accept Content-Type</name>
- </property>
- <property>
- <name>Follow Redirects</name>
- <value>false</value>
- </property>
- <property>
- <name>redirect-cookie-policy</name>
- <value>default</value>
- </property>
- <property>
- <name>proxy-configuration-service</name>
- </property>
- <property>
- <name>Proxy Host</name>
- </property>
- <property>
- <name>Proxy Port</name>
- </property>
- <property>
- <name>filename</name>
- <value>alexa-top-1m.csv</value>
- </property>
- </processor>
- <processor>
- <id>9d3d9047-fb85-3ae6-a815-0e19cc860c60</id>
- <name>Download Umbrella CSV file</name>
- <position x="-297.30227379373514" y="-61.444390100013806" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GetHTTP</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>1 day</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>URL</name>
- <value>http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip</value>
- </property>
- <property>
- <name>Filename</name>
- <value>umbrella-top-1m.csv.zip</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- </property>
- <property>
- <name>Username</name>
- </property>
- <property>
- <name>Password</name>
- </property>
- <property>
- <name>Connection Timeout</name>
- <value>30 sec</value>
- </property>
- <property>
- <name>Data Timeout</name>
- <value>10 min</value>
- </property>
- <property>
- <name>User Agent</name>
- </property>
- <property>
- <name>Accept Content-Type</name>
- </property>
- <property>
- <name>Follow Redirects</name>
- <value>false</value>
- </property>
- <property>
- <name>redirect-cookie-policy</name>
- <value>default</value>
- </property>
- <property>
- <name>proxy-configuration-service</name>
- </property>
- <property>
- <name>Proxy Host</name>
- </property>
- <property>
- <name>Proxy Port</name>
- </property>
- <property>
- <name>filename</name>
- <value>umbrella-top-1m.csv</value>
- </property>
- </processor>
- <processor>
- <id>9009320d-fb62-357e-ad94-bef8e95ea142</id>
- <name>Set filename</name>
- <position x="-294.78310875467264" y="388.04684866613775" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>filename</name>
- <value>${segment.original.filename}</value>
- </property>
- </processor>
- <processor>
- <id>86fdf574-d86b-3f35-9aa0-3ada1867aff8</id>
- <name>Add headers</name>
- <position x="325.04416175313986" y="201.70740433357992" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ReplaceText</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Regular Expression</name>
- <value>(?s)(^.*$)</value>
- </property>
- <property>
- <name>Replacement Value</name>
- <value>index,domain
-</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>Maximum Buffer Size</name>
- <value>1 MB</value>
- </property>
- <property>
- <name>Replacement Strategy</name>
- <value>Prepend</value>
- </property>
- <property>
- <name>Evaluation Mode</name>
- <value>Entire text</value>
- </property>
- <property>
- <name>Line-by-Line Evaluation Mode</name>
- <value>All</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <connection>
- <id>652026e5-0acd-3009-b45a-f68f3e37bef9</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>9d3d9047-fb85-3ae6-a815-0e19cc860c60</sourceId>
- <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>857cd537-4aeb-31fb-9740-0513e6cc46fe</destinationId>
- <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>bfe27587-bb06-388c-a59a-8aad9830cda1</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>3c4d65a9-aa39-380f-b16b-2aea028a019b</sourceId>
- <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>857cd537-4aeb-31fb-9740-0513e6cc46fe</destinationId>
- <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>7af7ff86-6b85-3fd1-bbc4-efa4e04593d9</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>857cd537-4aeb-31fb-9740-0513e6cc46fe</sourceId>
- <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>86fdf574-d86b-3f35-9aa0-3ada1867aff8</destinationId>
- <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>e2d60b76-d9bf-380f-9cfd-eeda1422ad73</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>86fdf574-d86b-3f35-9aa0-3ada1867aff8</sourceId>
- <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>9009320d-fb62-357e-ad94-bef8e95ea142</destinationId>
- <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>58b9bce4-6f7a-369c-a93f-dc23e252c670</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>9009320d-fb62-357e-ad94-bef8e95ea142</sourceId>
- <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>937de5fc-7d4a-35af-a071-46f04d6ea4fa</destinationId>
- <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>b997e46b-7905-33e8-8bbc-f4d51b0cc735</id>
- <name>Tor Nodes</name>
- <position x="968.7335178760902" y="456.9915202898361" />
- <comment>Downloads a CSV file of IP addresses used as Tor nodes</comment>
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>ad366a87-89d6-38ff-affe-a1f3575faa8a</id>
- <name>Save to disk</name>
- <position x="-328.58331298828125" y="-153.10000610351562" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.PutFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Directory</name>
- <value>/opt/nifi/nifi-current/conf/</value>
- </property>
- <property>
- <name>Conflict Resolution Strategy</name>
- <value>replace</value>
- </property>
- <property>
- <name>Create Missing Directories</name>
- <value>true</value>
- </property>
- <property>
- <name>Maximum File Count</name>
- </property>
- <property>
- <name>Last Modified Time</name>
- </property>
- <property>
- <name>Permissions</name>
- </property>
- <property>
- <name>Owner</name>
- </property>
- <property>
- <name>Group</name>
- </property>
- <autoTerminatedRelationship>success</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>34f52e1e-164e-34e4-b5fc-e5d16f773b19</id>
- <name>Get CSV file with Tor nodes</name>
- <position x="-323.0833282470703" y="-647.6000061035156" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GetHTTP</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>1 day</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>URL</name>
- <value>https://check.torproject.org/torbulkexitlist</value>
- </property>
- <property>
- <name>Filename</name>
- <value>tornodes.csv</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- <value>8972e39a-0176-1000-ffff-ffffb8dd96f4</value>
- </property>
- <property>
- <name>Username</name>
- </property>
- <property>
- <name>Password</name>
- </property>
- <property>
- <name>Connection Timeout</name>
- <value>30 sec</value>
- </property>
- <property>
- <name>Data Timeout</name>
- <value>30 sec</value>
- </property>
- <property>
- <name>User Agent</name>
- </property>
- <property>
- <name>Accept Content-Type</name>
- </property>
- <property>
- <name>Follow Redirects</name>
- <value>false</value>
- </property>
- <property>
- <name>redirect-cookie-policy</name>
- <value>default</value>
- </property>
- <property>
- <name>proxy-configuration-service</name>
- </property>
- <property>
- <name>Proxy Host</name>
- </property>
- <property>
- <name>Proxy Port</name>
- </property>
- </processor>
- <processor>
- <id>8c69ccb6-616f-3ce2-b0cd-57276cae3749</id>
- <name>Add header</name>
- <position x="-325.5833282470703" y="-410.1000061035156" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ReplaceText</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Regular Expression</name>
- <value>(?s)(^.*$)</value>
- </property>
- <property>
- <name>Replacement Value</name>
- <value>ip_addr
-</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>Maximum Buffer Size</name>
- <value>1 MB</value>
- </property>
- <property>
- <name>Replacement Strategy</name>
- <value>Prepend</value>
- </property>
- <property>
- <name>Evaluation Mode</name>
- <value>Entire text</value>
- </property>
- <property>
- <name>Line-by-Line Evaluation Mode</name>
- <value>All</value>
- </property>
- </processor>
- <connection>
- <id>33cb6d60-d003-3954-b9d0-f51ac40ed983</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</sourceId>
- <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>ad366a87-89d6-38ff-affe-a1f3575faa8a</destinationId>
- <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>cd13f619-bb19-37c2-b8fe-c962edfbf213</id>
- <name />
- <bendPoints>
- <bendPoint x="137.4166717529297" y="-370.1000061035156" />
- <bendPoint x="137.4166717529297" y="-320.1000061035156" />
- </bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</sourceId>
- <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</destinationId>
- <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>3941ee86-e740-3b8f-951a-c7da71e78fbe</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>34f52e1e-164e-34e4-b5fc-e5d16f773b19</sourceId>
- <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</destinationId>
- <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>19de0f5c-c244-3e7d-b711-ee165b493ea2</id>
- <name />
- <bendPoints>
- <bendPoint x="134.41668701171875" y="-113.10000610351562" />
- <bendPoint x="134.41668701171875" y="-63.100006103515625" />
- </bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>ad366a87-89d6-38ff-affe-a1f3575faa8a</sourceId>
- <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>ad366a87-89d6-38ff-affe-a1f3575faa8a</destinationId>
- <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>8130df3d-dc8c-32c2-975d-9c94438cac05</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>34f52e1e-164e-34e4-b5fc-e5d16f773b19</sourceId>
- <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</destinationId>
- <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>194a653f-0c92-3704-8bd4-ffa079643515</id>
- <name>Misp</name>
- <position x="548.9658647769079" y="453.4916238226681" />
- <comment>Polls Misp database once every minute and places new IOCs in a NiFi memcache.</comment>
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>283bea4b-2774-3f2d-aabe-cf96989e9997</id>
- <name>Set timestamp as FlowFile content</name>
- <position x="506.47715414708637" y="587.6551663734834" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.ReplaceText</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Regular Expression</name>
- <value>(?s)(^.*$)</value>
- </property>
- <property>
- <name>Replacement Value</name>
- <value>${timestamp}</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>Maximum Buffer Size</name>
- <value>1 MB</value>
- </property>
- <property>
- <name>Replacement Strategy</name>
- <value>Always Replace</value>
- </property>
- <property>
- <name>Evaluation Mode</name>
- <value>Entire text</value>
- </property>
- <property>
- <name>Line-by-Line Evaluation Mode</name>
- <value>All</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>e1e2caef-8178-3c91-b3ca-99f05f619064</id>
- <name>Get timestamp of last successful poll</name>
- <position x="-168.51082396716333" y="-293.9956980367642" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.FetchDistributedMapCache</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Cache Entry Identifier</name>
- <value>${lookup_id}</value>
- </property>
- <property>
- <name>Distributed Cache Service</name>
- <value>ad4d31bf-b1fb-35e0-b634-b969b200f3a6</value>
- </property>
- <property>
- <name>Put Cache Value In Attribute</name>
- <value>last_run</value>
- </property>
- <property>
- <name>Max Length To Put In Attribute</name>
- <value>256</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- </processor>
- <processor>
- <id>192802be-4416-3abc-ba03-90934f2df860</id>
- <name>Get events</name>
- <position x="-151.66592451726592" y="335.6012170464188" />
- <styles />
- <comment>Normally the query will have a filter at the end "/last:${last}" so that only new events are pulled. This has been removed from this demo.</comment>
- <class>org.apache.nifi.processors.standard.InvokeHTTP</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>HTTP Method</name>
- <value>GET</value>
- </property>
- <property>
- <name>Remote URL</name>
- <value>${misp_url}/attributes/restSearch/returnFormat:json/type:ip-src||ip-dst/last:${last}</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
- </property>
- <property>
- <name>Connection Timeout</name>
- <value>5 secs</value>
- </property>
- <property>
- <name>Read Timeout</name>
- <value>15 secs</value>
- </property>
- <property>
- <name>idle-timeout</name>
- <value>5 mins</value>
- </property>
- <property>
- <name>max-idle-connections</name>
- <value>5</value>
- </property>
- <property>
- <name>Include Date Header</name>
- <value>True</value>
- </property>
- <property>
- <name>Follow Redirects</name>
- <value>True</value>
- </property>
- <property>
- <name>Attributes to Send</name>
- </property>
- <property>
- <name>Useragent</name>
- </property>
- <property>
- <name>Basic Authentication Username</name>
- </property>
- <property>
- <name>Basic Authentication Password</name>
- </property>
- <property>
- <name>proxy-configuration-service</name>
- </property>
- <property>
- <name>Proxy Host</name>
- </property>
- <property>
- <name>Proxy Port</name>
- </property>
- <property>
- <name>Proxy Type</name>
- <value>http</value>
- </property>
- <property>
- <name>invokehttp-proxy-user</name>
- </property>
- <property>
- <name>invokehttp-proxy-password</name>
- </property>
- <property>
- <name>Put Response Body In Attribute</name>
- </property>
- <property>
- <name>Max Length To Put In Attribute</name>
- <value>256</value>
- </property>
- <property>
- <name>Digest Authentication</name>
- <value>false</value>
- </property>
- <property>
- <name>Always Output Response</name>
- <value>false</value>
- </property>
- <property>
- <name>Add Response Headers to Request</name>
- <value>false</value>
- </property>
- <property>
- <name>Content-Type</name>
- <value>${mime.type}</value>
- </property>
- <property>
- <name>send-message-body</name>
- <value>true</value>
- </property>
- <property>
- <name>Use Chunked Encoding</name>
- <value>false</value>
- </property>
- <property>
- <name>Penalize on "No Retry"</name>
- <value>false</value>
- </property>
- <property>
- <name>use-etag</name>
- <value>false</value>
- </property>
- <property>
- <name>etag-max-cache-size</name>
- <value>10MB</value>
- </property>
- <property>
- <name>ignore-response-content</name>
- <value>false</value>
- </property>
- <property>
- <name>form-body-form-name</name>
- </property>
- <property>
- <name>set-form-filename</name>
- <value>true</value>
- </property>
- <property>
- <name>Authorization</name>
- <value>${misp_token}</value>
- </property>
- <autoTerminatedRelationship>Original</autoTerminatedRelationship>
- <autoTerminatedRelationship>Failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>Retry</autoTerminatedRelationship>
- <autoTerminatedRelationship>No Retry</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>671c4e42-604f-389d-9cee-27431ca36448</id>
- <name>Store timestamp</name>
- <position x="504.4604101497308" y="824.0677052542044" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.PutDistributedMapCache</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Cache Entry Identifier</name>
- <value>${lookup_id}</value>
- </property>
- <property>
- <name>Distributed Cache Service</name>
- <value>ad4d31bf-b1fb-35e0-b634-b969b200f3a6</value>
- </property>
- <property>
- <name>Cache update strategy</name>
- <value>replace</value>
- </property>
- <property>
- <name>Max cache entry size</name>
- <value>1 MB</value>
- </property>
- <autoTerminatedRelationship>success</autoTerminatedRelationship>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>c7cc4e24-7d9a-3a17-8af1-ca655f46595f</id>
- <name>Update cache</name>
- <position x="-775.4735301448745" y="930.3624699197178" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.PutDistributedMapCache</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Cache Entry Identifier</name>
- <value>${misp_ip}</value>
- </property>
- <property>
- <name>Distributed Cache Service</name>
- <value>ad4d31bf-b1fb-35e0-b634-b969b200f3a6</value>
- </property>
- <property>
- <name>Cache update strategy</name>
- <value>replace</value>
- </property>
- <property>
- <name>Max cache entry size</name>
- <value>1 MB</value>
- </property>
- <autoTerminatedRelationship>success</autoTerminatedRelationship>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>81ec71a0-719a-3205-9360-6a535072f7c6</id>
- <name>Set attributes to get all events for the last x days</name>
- <position x="-378.1916613806792" y="-12.197472102501479" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>last</name>
- <value>${misp_first_interval}</value>
- </property>
- <property>
- <name>timestamp</name>
- <value>${now():toNumber()}</value>
- </property>
- </processor>
- <processor>
- <id>6d78b76c-5463-3610-b8c8-4796fa09c59b</id>
- <name>Periodic polling</name>
- <position x="-171.36520083798905" y="-518.6967632987289" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>1 minute</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>File Size</name>
- <value>0B</value>
- </property>
- <property>
- <name>Batch Size</name>
- <value>1</value>
- </property>
- <property>
- <name>Data Format</name>
- <value>Text</value>
- </property>
- <property>
- <name>Unique FlowFiles</name>
- <value>false</value>
- </property>
- <property>
- <name>generate-ff-custom-text</name>
- </property>
- <property>
- <name>character-set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>mime-type</name>
- </property>
- <property>
- <name>lookup_id</name>
- <value>ip</value>
- </property>
- </processor>
- <processor>
- <id>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</id>
- <name>Extract IP address</name>
- <position x="-156.69110558236184" y="543.7042207790005" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.EvaluateJsonPath</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Destination</name>
- <value>flowfile-attribute</value>
- </property>
- <property>
- <name>Return Type</name>
- <value>auto-detect</value>
- </property>
- <property>
- <name>Path Not Found Behavior</name>
- <value>ignore</value>
- </property>
- <property>
- <name>Null Value Representation</name>
- <value>empty string</value>
- </property>
- <property>
- <name>misp_ip</name>
- <value>$.value</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>unmatched</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>74d66e0e-0b65-36d2-96f1-4b836d2c4222</id>
- <name>Set attributes to get new events since last poll</name>
- <position x="81.93877074822706" y="-13.058372981407729" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-update-attribute-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Delete Attributes Expression</name>
- </property>
- <property>
- <name>Store State</name>
- <value>Do not store state</value>
- </property>
- <property>
- <name>Stateful Variables Initial Value</name>
- </property>
- <property>
- <name>canonical-value-lookup-cache-size</name>
- <value>100</value>
- </property>
- <property>
- <name>last</name>
- <value>${now():toNumber():minus(${last_run}):divide(60000):plus(1):append("m")}</value>
- </property>
- <property>
- <name>timestamp</name>
- <value>${now():toNumber()}</value>
- </property>
- </processor>
- <processor>
- <id>ba1b7e7e-a03c-3ace-9182-7f43569537e2</id>
- <name>Create one FlowFile for each IP address</name>
- <position x="-789.5267777615984" y="546.1428879861119" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.SplitJson</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>JsonPath Expression</name>
- <value>$.response.Attribute</value>
- </property>
- <property>
- <name>Null Value Representation</name>
- <value>empty string</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>d850fc04-df9a-36b7-b53f-8b397a1be69a</id>
- <name>Extract Misp event ID and store it to FlowFile</name>
- <position x="-783.5607955237681" y="719.2550630641567" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.EvaluateJsonPath</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Destination</name>
- <value>flowfile-content</value>
- </property>
- <property>
- <name>Return Type</name>
- <value>auto-detect</value>
- </property>
- <property>
- <name>Path Not Found Behavior</name>
- <value>ignore</value>
- </property>
- <property>
- <name>Null Value Representation</name>
- <value>empty string</value>
- </property>
- <property>
- <name>event_id</name>
- <value>$.event_id</value>
- </property>
- <autoTerminatedRelationship>failure</autoTerminatedRelationship>
- <autoTerminatedRelationship>unmatched</autoTerminatedRelationship>
- </processor>
- <connection>
- <id>39f7b787-0995-3721-8d50-700838b7a256</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>6d78b76c-5463-3610-b8c8-4796fa09c59b</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>e1e2caef-8178-3c91-b3ca-99f05f619064</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>adc3f55b-8d9a-33d0-a7af-0d795fa234ba</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>d850fc04-df9a-36b7-b53f-8b397a1be69a</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c7cc4e24-7d9a-3a17-8af1-ca655f46595f</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>matched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>4a797ab8-fb0e-3c9a-b397-b3394eca1ce4</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>ba1b7e7e-a03c-3ace-9182-7f43569537e2</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>split</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>4707ebf2-4b30-3e97-8abc-6ca8a9d168fd</id>
- <name>Consecutive poll</name>
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e1e2caef-8178-3c91-b3ca-99f05f619064</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>74d66e0e-0b65-36d2-96f1-4b836d2c4222</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>5e84f4ae-bf61-37d8-b115-0af74b89a6aa</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>d850fc04-df9a-36b7-b53f-8b397a1be69a</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>matched</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>58cc41df-404e-309b-9df6-2ea67e1fe2b7</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>283bea4b-2774-3f2d-aabe-cf96989e9997</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>671c4e42-604f-389d-9cee-27431ca36448</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>a9d8c7a2-6b55-3684-9954-92934d5a69e8</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>74d66e0e-0b65-36d2-96f1-4b836d2c4222</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>192802be-4416-3abc-ba03-90934f2df860</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>c022992b-534a-317a-943c-86142ee1cf81</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>81ec71a0-719a-3205-9360-6a535072f7c6</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>192802be-4416-3abc-ba03-90934f2df860</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>424c08f5-1ad1-3a0a-923c-c3fd988f7d2e</id>
- <name>Update timestamp</name>
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>192802be-4416-3abc-ba03-90934f2df860</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>283bea4b-2774-3f2d-aabe-cf96989e9997</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>Response</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>70fec2c7-4dd5-3dd1-92fa-59c3027bffb4</id>
- <name />
- <bendPoints>
- <bendPoint x="294.48917603283667" y="-253.99569803676422" />
- <bendPoint x="294.48917603283667" y="-203.99569803676422" />
- </bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e1e2caef-8178-3c91-b3ca-99f05f619064</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>e1e2caef-8178-3c91-b3ca-99f05f619064</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>d19116d2-0da0-3f86-8fd3-3285a839648e</id>
- <name>First poll</name>
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>e1e2caef-8178-3c91-b3ca-99f05f619064</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>81ec71a0-719a-3205-9360-6a535072f7c6</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>not-found</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>d375a69b-2139-3d9a-b6e3-48e0f69ec589</id>
- <name>Update cache with new events</name>
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>192802be-4416-3abc-ba03-90934f2df860</sourceId>
- <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>ba1b7e7e-a03c-3ace-9182-7f43569537e2</destinationId>
- <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>Response</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- </processGroup>
- <processGroup>
- <id>c4a200ea-5317-332a-97a4-ff76f951ecde</id>
- <name>GeoIP</name>
- <position x="556.427978515625" y="673.0274658203125" />
- <comment />
- <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
- <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
- <processor>
- <id>6292665b-f188-3551-b366-95476b5ac36f</id>
- <name>Save to disk</name>
- <position x="-357.78594755036767" y="656.471512008819" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.PutFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Directory</name>
- <value>/opt/nifi/nifi-current/conf/</value>
- </property>
- <property>
- <name>Conflict Resolution Strategy</name>
- <value>replace</value>
- </property>
- <property>
- <name>Create Missing Directories</name>
- <value>true</value>
- </property>
- <property>
- <name>Maximum File Count</name>
- </property>
- <property>
- <name>Last Modified Time</name>
- </property>
- <property>
- <name>Permissions</name>
- </property>
- <property>
- <name>Owner</name>
- </property>
- <property>
- <name>Group</name>
- </property>
- <autoTerminatedRelationship>success</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>c8b26516-0170-1000-ffff-fffffa357a77</id>
- <name>InvokeHTTP</name>
- <position x="-354.33263208075834" y="-1.6134650355261897" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.InvokeHTTP</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>HTTP Method</name>
- <value>GET</value>
- </property>
- <property>
- <name>Remote URL</name>
- <value>https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${maxmind_key}&suffix=tar.gz</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
- </property>
- <property>
- <name>Connection Timeout</name>
- <value>5 secs</value>
- </property>
- <property>
- <name>Read Timeout</name>
- <value>15 secs</value>
- </property>
- <property>
- <name>idle-timeout</name>
- <value>5 mins</value>
- </property>
- <property>
- <name>max-idle-connections</name>
- <value>5</value>
- </property>
- <property>
- <name>Include Date Header</name>
- <value>True</value>
- </property>
- <property>
- <name>Follow Redirects</name>
- <value>True</value>
- </property>
- <property>
- <name>Attributes to Send</name>
- </property>
- <property>
- <name>Useragent</name>
- </property>
- <property>
- <name>Basic Authentication Username</name>
- </property>
- <property>
- <name>Basic Authentication Password</name>
- </property>
- <property>
- <name>proxy-configuration-service</name>
- </property>
- <property>
- <name>Proxy Host</name>
- </property>
- <property>
- <name>Proxy Port</name>
- </property>
- <property>
- <name>Proxy Type</name>
- <value>http</value>
- </property>
- <property>
- <name>invokehttp-proxy-user</name>
- </property>
- <property>
- <name>invokehttp-proxy-password</name>
- </property>
- <property>
- <name>Put Response Body In Attribute</name>
- </property>
- <property>
- <name>Max Length To Put In Attribute</name>
- <value>256</value>
- </property>
- <property>
- <name>Digest Authentication</name>
- <value>false</value>
- </property>
- <property>
- <name>Always Output Response</name>
- <value>false</value>
- </property>
- <property>
- <name>Add Response Headers to Request</name>
- <value>false</value>
- </property>
- <property>
- <name>Content-Type</name>
- <value>${mime.type}</value>
- </property>
- <property>
- <name>send-message-body</name>
- <value>true</value>
- </property>
- <property>
- <name>Use Chunked Encoding</name>
- <value>false</value>
- </property>
- <property>
- <name>Penalize on "No Retry"</name>
- <value>false</value>
- </property>
- <property>
- <name>use-etag</name>
- <value>false</value>
- </property>
- <property>
- <name>etag-max-cache-size</name>
- <value>10MB</value>
- </property>
- <property>
- <name>ignore-response-content</name>
- <value>false</value>
- </property>
- <property>
- <name>form-body-form-name</name>
- </property>
- <property>
- <name>set-form-filename</name>
- <value>true</value>
- </property>
- <autoTerminatedRelationship>Original</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>b99eab15-7e38-33fa-87d1-41d772306d9c</id>
- <name>Uncompress</name>
- <position x="-359.13545011384423" y="239.87525101326742" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.CompressContent</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Mode</name>
- <value>decompress</value>
- </property>
- <property>
- <name>Compression Format</name>
- <value>use mime.type attribute</value>
- </property>
- <property>
- <name>Compression Level</name>
- <value>1</value>
- </property>
- <property>
- <name>Update Filename</name>
- <value>false</value>
- </property>
- </processor>
- <processor>
- <id>c8b20333-0170-1000-0000-000010760524</id>
- <name>RouteOnAttribute</name>
- <position x="-353.2358571852152" y="-223.16639543708658" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Routing Strategy</name>
- <value>Route to Property name</value>
- </property>
- <property>
- <name>maxmind_key</name>
- <value>${maxmind_key:length():gt(1)}</value>
- </property>
- <autoTerminatedRelationship>unmatched</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>aad91df7-8e80-3598-a3eb-9b000045b843</id>
- <name>UnpackContent</name>
- <position x="-358.13545011384423" y="448.82544805040084" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.UnpackContent</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>0 sec</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>Packaging Format</name>
- <value>tar</value>
- </property>
- <property>
- <name>File Filter</name>
- <value>GeoLite2-City.mmdb</value>
- </property>
- <autoTerminatedRelationship>original</autoTerminatedRelationship>
- </processor>
- <processor>
- <id>c8b1bafd-0170-1000-0000-0000753f5f5b</id>
- <name>GenerateFlowFile</name>
- <position x="-366.3974570271698" y="-455.687252544095" />
- <styles />
- <comment />
- <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-standard-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <maxConcurrentTasks>1</maxConcurrentTasks>
- <schedulingPeriod>1 week</schedulingPeriod>
- <penalizationPeriod>30 sec</penalizationPeriod>
- <yieldPeriod>1 sec</yieldPeriod>
- <bulletinLevel>WARN</bulletinLevel>
- <lossTolerant>false</lossTolerant>
- <scheduledState>RUNNING</scheduledState>
- <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
- <executionNode>ALL</executionNode>
- <runDurationNanos>0</runDurationNanos>
- <property>
- <name>File Size</name>
- <value>0B</value>
- </property>
- <property>
- <name>Batch Size</name>
- <value>1</value>
- </property>
- <property>
- <name>Data Format</name>
- <value>Text</value>
- </property>
- <property>
- <name>Unique FlowFiles</name>
- <value>false</value>
- </property>
- <property>
- <name>generate-ff-custom-text</name>
- </property>
- <property>
- <name>character-set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>mime-type</name>
- </property>
- </processor>
- <funnel>
- <id>c2cac6f3-c926-3038-b685-68f71f76fda3</id>
- <position x="457.8712158203125" y="380.06201171875" />
- </funnel>
- <connection>
- <id>c8b21bba-0170-1000-0000-0000281b44ba</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>c8b1bafd-0170-1000-0000-0000753f5f5b</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c8b20333-0170-1000-0000-000010760524</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>1d0b1e6f-7b01-34c5-82f8-c95918e700ae</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>b99eab15-7e38-33fa-87d1-41d772306d9c</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>FUNNEL</destinationType>
- <relationship>failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>e8e86b3f-6936-3080-8eb6-036d532cb483</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>6292665b-f188-3551-b366-95476b5ac36f</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>FUNNEL</destinationType>
- <relationship>failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>e2f43878-959f-379c-b898-6d7c3a72af44</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>b99eab15-7e38-33fa-87d1-41d772306d9c</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>aad91df7-8e80-3598-a3eb-9b000045b843</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
+ <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>10 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>File Size</name>
+ <value>0B</value>
+ </property>
+ <property>
+ <name>Batch Size</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>Data Format</name>
+ <value>Text</value>
+ </property>
+ <property>
+ <name>Unique FlowFiles</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>generate-ff-custom-text</name>
+ <value>[{"timestamp":"${now():format('yyyy-MM-dd HH:mm:ss.SSS'):replaceFirst(' ','T')}","TLP":"AMBER","destination":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":443},"event_type":"tls","flow_id":852792667052212,"in_iface":"if1","proto":"TCP","source":{"ip":"10.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}.${random():mod(254):plus(1)}","port":53466},"tls":{"ja3":{"hash":"e5b607b5862a46cab44d7bacd582b3cd","string":"771,4867-4865-4866-52393-52392-49195-49199-49196-49200-49171-49172-156-157-47-53-10,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0"},"sni":"clients3.google.com","sni_domain":"google.com","version":"TLS 1.3"}}]</value>
+ </property>
+ <property>
+ <name>character-set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>mime.type</name>
+ <value>application/json</value>
+ </property>
+ <property>
+ <name>enrich_domain1</name>
+ <value>/tls/sni_domain</value>
+ </property>
+ <property>
+ <name>enrich_fqdn1</name>
+ <value>/tls/sni</value>
+ </property>
+ <property>
+ <name>enrich_ip1</name>
+ <value>/source/ip</value>
+ </property>
+ <property>
+ <name>data_id</name>
+ <value>suricata_tls</value>
+ </property>
+ <property>
+ <name>data_index</name>
+ <value>logs-suricata-tls</value>
+ </property>
+ <property>
+ <name>data_type</name>
+ <value>suricata</value>
+ </property>
+ <property>
+ <name>enrich_ip2</name>
+ <value>/destination/ip</value>
+ </property>
+ </processor>
+ <outputPort>
+ <id>0c864b15-0175-1000-0000-00001d403b1e</id>
+ <name>To enrichment</name>
+ <position x="1192.0" y="576.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </outputPort>
+ <connection>
+ <id>1cb3658c-0175-1000-ffff-ffff93193081</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>f8143c19-b547-1c84-90b7-2e3c37a659e7</sourceId>
+ <sourceGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>0c864b15-0175-1000-0000-00001d403b1e</destinationId>
+ <destinationGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>0c865d01-0175-1000-0000-0000559b408d</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>bb7dc9ff-2d25-3134-9617-cca3cabe9179</sourceId>
+ <sourceGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>0c864b15-0175-1000-0000-00001d403b1e</destinationId>
+ <destinationGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <connection>
+ <id>0c866fa6-0175-1000-ffff-ffffe866c936</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>0c864b15-0175-1000-0000-00001d403b1e</sourceId>
+ <sourceGroupId>0c83ef26-0175-1000-ffff-ffffcac37910</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>d30dc946-251a-307c-8e88-f2262b0bb194</destinationId>
+ <destinationGroupId>b3d57504-7c06-37a3-b59b-8723f60fa728</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
<connection>
- <id>dc7524de-fb6c-3e02-8c60-f821d81aff29</id>
+ <id>c5fe676f-baa5-3d90-956e-fe502db0ac68</id>
<name />
<bendPoints />
- <labelIndex>1</labelIndex>
+ <labelIndex>0</labelIndex>
<zIndex>0</zIndex>
- <sourceId>aad91df7-8e80-3598-a3eb-9b000045b843</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>6292665b-f188-3551-b366-95476b5ac36f</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>success</relationship>
+ <sourceId>d30dc946-251a-307c-8e88-f2262b0bb194</sourceId>
+ <sourceGroupId>b3d57504-7c06-37a3-b59b-8723f60fa728</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>20b01ab3-3a8d-3573-b95d-a4a45494050f</destinationId>
+ <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -10960,18 +5770,18 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>39a11e18-3397-3f1a-a020-49b895ff6f81</id>
+ <id>214d5013-0175-1000-ffff-ffff9b7dbebb</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
- <sourceId>aad91df7-8e80-3598-a3eb-9b000045b843</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>FUNNEL</destinationType>
- <relationship>failure</relationship>
+ <sourceId>1ef39440-1985-3bbb-8e03-859a1c5ee4b1</sourceId>
+ <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>21a9e277-2d80-359a-9c57-cb76d8962e6d</destinationId>
+ <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -10980,82 +5790,20 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
- <id>c8b2e58b-0170-1000-ffff-ffff997c6e6c</id>
+ <id>27d64272-0172-1000-0000-000079e1c9c6</id>
<name />
<bendPoints>
- <bendPoint x="108.66736791924166" y="38.38653496447381" />
- <bendPoint x="136.0" y="88.0" />
+ <bendPoint x="88.0" y="864.0" />
</bendPoints>
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>c8b26516-0170-1000-ffff-fffffa357a77</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c8b26516-0170-1000-ffff-fffffa357a77</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>Retry</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>c8b29bee-0170-1000-ffff-fffff516df5d</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>c8b26516-0170-1000-ffff-fffffa357a77</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>b99eab15-7e38-33fa-87d1-41d772306d9c</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>Response</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>c8b2cb01-0170-1000-0000-000005baadda</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
- <zIndex>0</zIndex>
- <sourceId>c8b26516-0170-1000-ffff-fffffa357a77</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>FUNNEL</destinationType>
- <relationship>No Retry</relationship>
- <relationship>Failure</relationship>
- <maxWorkQueueSize>10000</maxWorkQueueSize>
- <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
- <flowFileExpiration>0 sec</flowFileExpiration>
- <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
- <partitioningAttribute />
- <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
- </connection>
- <connection>
- <id>c8b5c90e-0170-1000-ffff-ffff9864e7e4</id>
- <name />
- <bendPoints />
- <labelIndex>1</labelIndex>
+ <labelIndex>0</labelIndex>
<zIndex>0</zIndex>
- <sourceId>c8b20333-0170-1000-0000-000010760524</sourceId>
- <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
- <sourceType>PROCESSOR</sourceType>
- <destinationId>c8b26516-0170-1000-ffff-fffffa357a77</destinationId>
- <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
- <destinationType>PROCESSOR</destinationType>
- <relationship>maxmind_key</relationship>
+ <sourceId>27d5761b-0172-1000-0000-000059275dad</sourceId>
+ <sourceGroupId>27d51d04-0172-1000-0000-00004573c6ec</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>20b01ab3-3a8d-3573-b95d-a4a45494050f</destinationId>
+ <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
@@ -11063,436 +5811,2776 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
- </processGroup>
- </processGroup>
- <controllerService>
- <id>349b34c7-a821-1197-ffff-ffff85d82877</id>
- <name>Contry code to region</name>
- <comment />
- <class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-lookup-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>csv-file</name>
- <value>/opt/nifi/nifi-current/conf/enrich/CountriesWithRegionalCodes.csv</value>
- </property>
- <property>
- <name>CSV Format</name>
- <value>default</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>lookup-key-column</name>
- <value>alpha-2</value>
- </property>
- <property>
- <name>ignore-duplicates</name>
- <value>true</value>
- </property>
- <property>
- <name>Value Separator</name>
- <value>,</value>
- </property>
- <property>
- <name>Quote Character</name>
- <value>"</value>
- </property>
- <property>
- <name>Quote Mode</name>
- <value>MINIMAL</value>
- </property>
- <property>
- <name>Comment Marker</name>
- </property>
- <property>
- <name>Escape Character</name>
- <value>\</value>
- </property>
- <property>
- <name>Trim Fields</name>
- <value>true</value>
- </property>
- <property>
- <name>lookup-value-column</name>
- <value>region</value>
- </property>
- </controllerService>
- <controllerService>
- <id>8972e39a-0176-1000-ffff-ffffb8dd96f4</id>
- <name>Common CA</name>
- <comment />
- <class>org.apache.nifi.ssl.StandardSSLContextService</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-ssl-context-service-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>Keystore Filename</name>
- </property>
- <property>
- <name>Keystore Password</name>
- </property>
- <property>
- <name>key-password</name>
- </property>
- <property>
- <name>Keystore Type</name>
- </property>
- <property>
- <name>Truststore Filename</name>
- <value>/opt/nifi/nifi-current/conf/common-cacerts.jks</value>
- </property>
- <property>
- <name>Truststore Password</name>
- <value>enc{2650a175fb2f75e2dcd038b4b506ac6368b7e025f6cb80fa6a82b187b0755443}</value>
- </property>
- <property>
- <name>Truststore Type</name>
- <value>JKS</value>
- </property>
- <property>
- <name>SSL Protocol</name>
- <value>TLS</value>
- </property>
- </controllerService>
- <controllerService>
- <id>bbd4d3a2-0175-1000-0000-00000b0fb8bd</id>
- <name>Tor node CSV</name>
- <comment />
- <class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-lookup-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>csv-file</name>
- <value>/opt/nifi/nifi-current/conf/enrich/tornodes.csv</value>
- </property>
- <property>
- <name>CSV Format</name>
- <value>default</value>
- </property>
- <property>
- <name>Character Set</name>
- <value>UTF-8</value>
- </property>
- <property>
- <name>lookup-key-column</name>
- <value>ip_addr</value>
- </property>
- <property>
- <name>ignore-duplicates</name>
- <value>true</value>
- </property>
- <property>
- <name>Value Separator</name>
- <value>,</value>
- </property>
- <property>
- <name>Quote Character</name>
- <value>"</value>
- </property>
- <property>
- <name>Quote Mode</name>
- <value>MINIMAL</value>
- </property>
- <property>
- <name>Comment Marker</name>
- </property>
- <property>
- <name>Escape Character</name>
- <value>\</value>
- </property>
- <property>
- <name>Trim Fields</name>
- <value>true</value>
- </property>
- <property>
- <name>lookup-value-column</name>
- <value>ip_addr</value>
- </property>
- </controllerService>
- <controllerService>
- <id>14453a95-7646-1485-0000-00002c675762</id>
- <name>Mysql audit log</name>
- <comment />
- <class>org.apache.nifi.csv.CSVReader</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>schema-access-strategy</name>
- <value>infer-schema</value>
- </property>
- <property>
- <name>schema-registry</name>
- </property>
- <property>
- <name>schema-name</name>
- <value>${schema.name}</value>
- </property>
- <property>
- <name>schema-version</name>
- </property>
- <property>
- <name>schema-branch</name>
- </property>
- <property>
- <name>schema-text</name>
- <value>${avro.schema}</value>
- </property>
- <property>
- <name>csv-reader-csv-parser</name>
- <value>commons-csv</value>
- </property>
- <property>
- <name>Date Format</name>
- </property>
- <property>
- <name>Time Format</name>
- </property>
- <property>
- <name>Timestamp Format</name>
- </property>
- <property>
- <name>CSV Format</name>
- <value>custom</value>
- </property>
- <property>
- <name>Value Separator</name>
- <value>,</value>
- </property>
- <property>
- <name>Record Separator</name>
- <value>\n</value>
- </property>
- <property>
- <name>Skip Header Line</name>
- <value>true</value>
- </property>
- <property>
- <name>ignore-csv-header</name>
- <value>false</value>
- </property>
- <property>
- <name>Quote Character</name>
- <value>"</value>
- </property>
- <property>
- <name>Escape Character</name>
- <value>\</value>
- </property>
- <property>
- <name>Comment Marker</name>
- </property>
- <property>
- <name>Null String</name>
- </property>
- <property>
- <name>Trim Fields</name>
- <value>true</value>
- </property>
- <property>
- <name>csvutils-character-set</name>
- <value>UTF-8</value>
- </property>
- </controllerService>
- <controllerService>
- <id>7504a565-0176-1000-ffff-ffff9c0f0741</id>
- <name>Zookeeper logs</name>
- <comment />
- <class>org.apache.nifi.grok.GrokReader</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>schema-access-strategy</name>
- <value>string-fields-from-grok-expression</value>
- </property>
- <property>
- <name>schema-registry</name>
- </property>
- <property>
- <name>schema-name</name>
- <value>${schema.name}</value>
- </property>
- <property>
- <name>schema-version</name>
- </property>
- <property>
- <name>schema-branch</name>
- </property>
- <property>
- <name>schema-text</name>
- <value>${avro.schema}</value>
- </property>
- <property>
- <name>Grok Pattern File</name>
- </property>
- <property>
- <name>Grok Expression</name>
- <value>%{GREEDYDATA:timestamp} \[%{DATA:id}\] - %{DATA:level} \[%{DATA:process}\] - %{GREEDYDATA:message}</value>
- </property>
- <property>
- <name>no-match-behavior</name>
- <value>append-to-previous-message</value>
- </property>
- </controllerService>
- <controllerService>
- <id>8b1dd8bb-0170-1000-0000-000007446e6a</id>
- <name>Misp DistributedMapCacheServer</name>
- <comment />
- <class>org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-distributed-cache-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>Port</name>
- <value>4557</value>
- </property>
- <property>
- <name>Maximum Cache Entries</name>
- <value>10000</value>
- </property>
- <property>
- <name>Eviction Strategy</name>
- <value>Least Frequently Used</value>
- </property>
- <property>
- <name>Persistence Directory</name>
- <value>/opt/nifi/nifi-current/conf/</value>
- </property>
- <property>
- <name>SSL Context Service</name>
- </property>
- </controllerService>
- <controllerService>
- <id>56ebe0aa-0176-1000-ffff-ffffbd212f01</id>
- <name>Haproxy GrokReader</name>
+ <connection>
+ <id>27d65fe7-0172-1000-ffff-ffffec2db03b</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-744.0" y="856.0" />
+ </bendPoints>
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>27d5dab2-0172-1000-ffff-ffffab5c50be</sourceId>
+ <sourceGroupId>27d51d04-0172-1000-0000-00004573c6ec</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>21a9e277-2d80-359a-9c57-cb76d8962e6d</destinationId>
+ <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
+ <destinationType>OUTPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <controllerService>
+ <id>94600c6c-704e-3ff8-a2a4-f2f25c71dc3b</id>
+ <name>JsonRecordSetWriter</name>
+ <comment />
+ <class>org.apache.nifi.json.JsonRecordSetWriter</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-record-serialization-services-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <enabled>true</enabled>
+ <property>
+ <name>Schema Write Strategy</name>
+ <value>no-schema</value>
+ </property>
+ <property>
+ <name>schema-cache</name>
+ </property>
+ <property>
+ <name>schema-access-strategy</name>
+ <value>inherit-record-schema</value>
+ </property>
+ <property>
+ <name>schema-registry</name>
+ </property>
+ <property>
+ <name>schema-name</name>
+ <value>${schema.name}</value>
+ </property>
+ <property>
+ <name>schema-version</name>
+ </property>
+ <property>
+ <name>schema-branch</name>
+ </property>
+ <property>
+ <name>schema-text</name>
+ <value>${avro.schema}</value>
+ </property>
+ <property>
+ <name>Date Format</name>
+ </property>
+ <property>
+ <name>Time Format</name>
+ </property>
+ <property>
+ <name>Timestamp Format</name>
+ </property>
+ <property>
+ <name>Pretty Print JSON</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>suppress-nulls</name>
+ <value>suppress-missing</value>
+ </property>
+ <property>
+ <name>output-grouping</name>
+ <value>output-array</value>
+ </property>
+ <property>
+ <name>compression-format</name>
+ <value>none</value>
+ </property>
+ <property>
+ <name>compression-level</name>
+ <value>1</value>
+ </property>
+ </controllerService>
+ <controllerService>
+ <id>09b4fa02-0459-358d-939f-54fda8aea702</id>
+ <name>VolatileSchemaCache</name>
+ <comment />
+ <class>org.apache.nifi.schema.inference.VolatileSchemaCache</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-record-serialization-services-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <enabled>false</enabled>
+ <property>
+ <name>max-cache-size</name>
+ <value>100</value>
+ </property>
+ </controllerService>
+ </processGroup>
+ <processGroup>
+ <id>e9c19adc-c8a4-327e-ad24-24e71fd3474e</id>
+ <name>Data output</name>
+ <position x="829.4446253936723" y="1015.2711478364996" />
+ <comment />
+ <inputPort>
+ <id>e333b82d-7408-3747-8dd2-46473704e51b</id>
+ <name>Data input</name>
+ <position x="-688.0" y="496.0" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </inputPort>
+ <processGroup>
+ <id>7ebf304b-4978-3adc-ac31-470fb76e5029</id>
+ <name>Elastic odfe</name>
+ <position x="-759.1319580078125" y="739.6137390136719" />
+ <comment />
+ <processor>
+ <id>8b48f28f-2379-3f0f-81fe-4e1b93e72666</id>
+ <name>PutElasticsearchHttpRecord</name>
+ <position x="-856.2311706542969" y="629.8186340332031" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.elasticsearch.PutElasticsearchHttpRecord</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-elasticsearch-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>elasticsearch-http-url</name>
+ <value>${elastic_url}</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Username</name>
+ <value>${elastic_username}</value>
+ </property>
+ <property>
+ <name>Password</name>
+ <value>enc{aa0e200e6ad20acb3eb1e1f1c7ab08154fc11ccf55c6176c4c8b12fab9f339cba76c4cf1f567bb8aeb4802017cc50639}</value>
+ </property>
+ <property>
+ <name>elasticsearch-http-connect-timeout</name>
+ <value>5 secs</value>
+ </property>
+ <property>
+ <name>elasticsearch-http-response-timeout</name>
+ <value>15 secs</value>
+ </property>
+ <property>
+ <name>proxy-configuration-service</name>
+ </property>
+ <property>
+ <name>elasticsearch-http-proxy-host</name>
+ </property>
+ <property>
+ <name>elasticsearch-http-proxy-port</name>
+ </property>
+ <property>
+ <name>proxy-username</name>
+ </property>
+ <property>
+ <name>proxy-password</name>
+ </property>
+ <property>
+ <name>put-es-record-record-reader</name>
+ <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
+ </property>
+ <property>
+ <name>put-es-record-record-writer</name>
+ </property>
+ <property>
+ <name>put-es-record-log-all-errors</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>put-es-record-id-path</name>
+ </property>
+ <property>
+ <name>put-es-record-index</name>
+ <value>${data_index}-${now():format("yyyy-MM-dd")}</value>
+ </property>
+ <property>
+ <name>put-es-record-type</name>
+ <value>_doc</value>
+ </property>
+ <property>
+ <name>put-es-record-index-op</name>
+ <value>index</value>
+ </property>
+ <property>
+ <name>suppress-nulls</name>
+ <value>always-suppress</value>
+ </property>
+ <property>
+ <name>Date Format</name>
+ </property>
+ <property>
+ <name>Time Format</name>
+ </property>
+ <property>
+ <name>Timestamp Format</name>
+ </property>
+ <autoTerminatedRelationship>success</autoTerminatedRelationship>
+ </processor>
+ <inputPort>
+ <id>e7d34e01-babe-3022-ad9b-a7620e7c0f38</id>
+ <name>Data input</name>
+ <position x="-803.9990234375" y="484.5271301269531" />
+ <comments />
+ <scheduledState>RUNNING</scheduledState>
+ </inputPort>
+ <funnel>
+ <id>a8cf8491-c2a7-3986-b803-58aff43326de</id>
+ <position x="-709.0761208187066" y="911.6861746431973" />
+ </funnel>
+ <connection>
+ <id>3280c550-2117-37a6-8b5e-3bc1953fa17e</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-393.2311706542969" y="669.8186340332031" />
+ <bendPoint x="-393.2311706542969" y="719.8186340332031" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</sourceId>
+ <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId>
+ <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>retry</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>02a9e341-0590-34a8-9f0c-9d6992869e59</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</sourceId>
+ <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>a8cf8491-c2a7-3986-b803-58aff43326de</destinationId>
+ <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>1 min</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5de8f98f-ce46-3565-b0ce-7f8ecf518c53</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</sourceId>
+ <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId>
+ <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>10 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <processGroup>
+ <id>2bb2f914-0172-1000-0000-0000240c76e4</id>
+ <name>Custom output</name>
+ <position x="-328.0" y="744.0" />
+ <comment />
+ <inputPort>
+ <id>2bb31aa5-0172-1000-0000-00000869fb70</id>
+ <name>Input</name>
+ <position x="-648.0" y="496.0" />
+ <comments />
+ <scheduledState>STOPPED</scheduledState>
+ </inputPort>
+ </processGroup>
+ <connection>
+ <id>34772170-2400-3eb6-b9c5-c03b912a38f3</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e333b82d-7408-3747-8dd2-46473704e51b</sourceId>
+ <sourceGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</sourceGroupId>
+ <sourceType>INPUT_PORT</sourceType>
+ <destinationId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</destinationId>
+ <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <connection>
+ <id>875a975e-46e1-36fa-a035-4799201abd63</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>20b01ab3-3a8d-3573-b95d-a4a45494050f</sourceId>
+ <sourceGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>74abf119-faa6-3e9e-bb31-da2e79f89a38</destinationId>
+ <destinationGroupId>fcbcacd1-542d-3a15-a5aa-9c1302328954</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>bbc37560-0171-1000-0000-000055178fff</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c164884d-277f-31af-ac3c-18b211667bbf</sourceId>
+ <sourceGroupId>fcbcacd1-542d-3a15-a5aa-9c1302328954</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>e333b82d-7408-3747-8dd2-46473704e51b</destinationId>
+ <destinationGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>9cdaaee8-0e39-3dbd-a7cc-06a89056bb7c</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>21a9e277-2d80-359a-9c57-cb76d8962e6d</sourceId>
+ <sourceGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</sourceGroupId>
+ <sourceType>OUTPUT_PORT</sourceType>
+ <destinationId>e333b82d-7408-3747-8dd2-46473704e51b</destinationId>
+ <destinationGroupId>e9c19adc-c8a4-327e-ad24-24e71fd3474e</destinationGroupId>
+ <destinationType>INPUT_PORT</destinationType>
+ <relationship />
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <controllerService>
+ <id>b7794eb3-9227-36dd-8751-e87d1c2321ee</id>
+ <name>Misp DistributedMapCacheClientService</name>
+ <comment />
+ <class>org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-distributed-cache-services-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <enabled>false</enabled>
+ <property>
+ <name>Server Hostname</name>
+ <value>localhost</value>
+ </property>
+ <property>
+ <name>Server Port</name>
+ <value>6000</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ </property>
+ <property>
+ <name>Communications Timeout</name>
+ <value>30 secs</value>
+ </property>
+ </controllerService>
+ </processGroup>
+ <processGroup>
+ <id>72eb009e-0c2f-302d-bc6c-2d02c29c25a9</id>
+ <name>Enrichment data</name>
+ <position x="1720.0" y="248.0" />
<comment />
- <class>org.apache.nifi.grok.GrokReader</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>schema-access-strategy</name>
- <value>string-fields-from-grok-expression</value>
- </property>
- <property>
- <name>schema-registry</name>
- </property>
- <property>
- <name>schema-name</name>
- <value>${schema.name}</value>
- </property>
- <property>
- <name>schema-version</name>
- </property>
- <property>
- <name>schema-branch</name>
- </property>
- <property>
- <name>schema-text</name>
- <value>${avro.schema}</value>
- </property>
- <property>
- <name>Grok Pattern File</name>
- <value>/opt/nifi/nifi-current/conf/enrich/haproxy.groklib</value>
- </property>
- <property>
- <name>Grok Expression</name>
- <value>%{PROG:process.name}(?:\[%{POSINT:process.pid}\])?: %{HAPROXYHTTPBASE}</value>
- </property>
- <property>
- <name>no-match-behavior</name>
- <value>append-to-previous-message</value>
- </property>
- </controllerService>
+ <processGroup>
+ <id>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</id>
+ <name>Top domains</name>
+ <position x="970.3727876614566" y="673.4981494769316" />
+ <comment>Downloads CSV files containing top domains from Alexa and Umbrella</comment>
+ <processor>
+ <id>857cd537-4aeb-31fb-9740-0513e6cc46fe</id>
+ <name>Unzip CSV files</name>
+ <position x="-297.30227379373514" y="212.70767899178307" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.UnpackContent</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Packaging Format</name>
+ <value>zip</value>
+ </property>
+ <property>
+ <name>File Filter</name>
+ <value>.*</value>
+ </property>
+ <autoTerminatedRelationship>original</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>937de5fc-7d4a-35af-a071-46f04d6ea4fa</id>
+ <name>Save to disk</name>
+ <position x="326.18698401876486" y="392.4228279175642" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.PutFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Directory</name>
+ <value>/opt/nifi/nifi-current/conf/</value>
+ </property>
+ <property>
+ <name>Conflict Resolution Strategy</name>
+ <value>replace</value>
+ </property>
+ <property>
+ <name>Create Missing Directories</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Maximum File Count</name>
+ </property>
+ <property>
+ <name>Last Modified Time</name>
+ </property>
+ <property>
+ <name>Permissions</name>
+ </property>
+ <property>
+ <name>Owner</name>
+ </property>
+ <property>
+ <name>Group</name>
+ </property>
+ <autoTerminatedRelationship>success</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>3c4d65a9-aa39-380f-b16b-2aea028a019b</id>
+ <name>Download Alexa CSV file</name>
+ <position x="197.54468055196799" y="-60.57735518790443" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.GetHTTP</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>1 day</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>URL</name>
+ <value>http://s3.amazonaws.com/alexa-static/top-1m.csv.zip</value>
+ </property>
+ <property>
+ <name>Filename</name>
+ <value>alexa-top-1m.csv.zip</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ </property>
+ <property>
+ <name>Username</name>
+ </property>
+ <property>
+ <name>Password</name>
+ </property>
+ <property>
+ <name>Connection Timeout</name>
+ <value>30 sec</value>
+ </property>
+ <property>
+ <name>Data Timeout</name>
+ <value>10 min</value>
+ </property>
+ <property>
+ <name>User Agent</name>
+ </property>
+ <property>
+ <name>Accept Content-Type</name>
+ </property>
+ <property>
+ <name>Follow Redirects</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>redirect-cookie-policy</name>
+ <value>default</value>
+ </property>
+ <property>
+ <name>proxy-configuration-service</name>
+ </property>
+ <property>
+ <name>Proxy Host</name>
+ </property>
+ <property>
+ <name>Proxy Port</name>
+ </property>
+ <property>
+ <name>filename</name>
+ <value>alexa-top-1m.csv</value>
+ </property>
+ </processor>
+ <processor>
+ <id>9d3d9047-fb85-3ae6-a815-0e19cc860c60</id>
+ <name>Download Umbrella CSV file</name>
+ <position x="-297.30227379373514" y="-61.444390100013806" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.GetHTTP</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>1 day</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>URL</name>
+ <value>http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip</value>
+ </property>
+ <property>
+ <name>Filename</name>
+ <value>umbrella-top-1m.csv.zip</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ </property>
+ <property>
+ <name>Username</name>
+ </property>
+ <property>
+ <name>Password</name>
+ </property>
+ <property>
+ <name>Connection Timeout</name>
+ <value>30 sec</value>
+ </property>
+ <property>
+ <name>Data Timeout</name>
+ <value>10 min</value>
+ </property>
+ <property>
+ <name>User Agent</name>
+ </property>
+ <property>
+ <name>Accept Content-Type</name>
+ </property>
+ <property>
+ <name>Follow Redirects</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>redirect-cookie-policy</name>
+ <value>default</value>
+ </property>
+ <property>
+ <name>proxy-configuration-service</name>
+ </property>
+ <property>
+ <name>Proxy Host</name>
+ </property>
+ <property>
+ <name>Proxy Port</name>
+ </property>
+ <property>
+ <name>filename</name>
+ <value>umbrella-top-1m.csv</value>
+ </property>
+ </processor>
+ <processor>
+ <id>9009320d-fb62-357e-ad94-bef8e95ea142</id>
+ <name>Set filename</name>
+ <position x="-294.78310875467264" y="388.04684866613775" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>filename</name>
+ <value>${segment.original.filename}</value>
+ </property>
+ </processor>
+ <processor>
+ <id>86fdf574-d86b-3f35-9aa0-3ada1867aff8</id>
+ <name>Add headers</name>
+ <position x="325.04416175313986" y="201.70740433357992" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ReplaceText</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Regular Expression</name>
+ <value>(?s)(^.*$)</value>
+ </property>
+ <property>
+ <name>Replacement Value</name>
+ <value>index,domain
+</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Maximum Buffer Size</name>
+ <value>1 MB</value>
+ </property>
+ <property>
+ <name>Replacement Strategy</name>
+ <value>Prepend</value>
+ </property>
+ <property>
+ <name>Evaluation Mode</name>
+ <value>Entire text</value>
+ </property>
+ <property>
+ <name>Line-by-Line Evaluation Mode</name>
+ <value>All</value>
+ </property>
+ </processor>
+ <connection>
+ <id>652026e5-0acd-3009-b45a-f68f3e37bef9</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>9d3d9047-fb85-3ae6-a815-0e19cc860c60</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>857cd537-4aeb-31fb-9740-0513e6cc46fe</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>b400d4e7-7106-3ea3-8b1a-0b9d2a8795b2</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="787.0441617531399" y="276.7074043335799" />
+ <bendPoint x="788.0441617531399" y="291.7074043335799" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>86fdf574-d86b-3f35-9aa0-3ada1867aff8</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>86fdf574-d86b-3f35-9aa0-3ada1867aff8</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5d7b82fa-10f8-3a32-9ffa-ebce53eb6070</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-414.74468712381326" y="221.65236588143148" />
+ <bendPoint x="-451.48125938943826" y="275.1232673706893" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>857cd537-4aeb-31fb-9740-0513e6cc46fe</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>857cd537-4aeb-31fb-9740-0513e6cc46fe</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>bfe27587-bb06-388c-a59a-8aad9830cda1</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>3c4d65a9-aa39-380f-b16b-2aea028a019b</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>857cd537-4aeb-31fb-9740-0513e6cc46fe</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>7af7ff86-6b85-3fd1-bbc4-efa4e04593d9</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>857cd537-4aeb-31fb-9740-0513e6cc46fe</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>86fdf574-d86b-3f35-9aa0-3ada1867aff8</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>e2d60b76-d9bf-380f-9cfd-eeda1422ad73</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>86fdf574-d86b-3f35-9aa0-3ada1867aff8</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>9009320d-fb62-357e-ad94-bef8e95ea142</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>58b9bce4-6f7a-369c-a93f-dc23e252c670</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>9009320d-fb62-357e-ad94-bef8e95ea142</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>937de5fc-7d4a-35af-a071-46f04d6ea4fa</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>dea956ed-2b3b-39de-8cd8-a4d1f7a88aa2</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="790.1869840187649" y="473.4228279175642" />
+ <bendPoint x="789.1869840187649" y="482.42282791756406" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>937de5fc-7d4a-35af-a071-46f04d6ea4fa</sourceId>
+ <sourceGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>937de5fc-7d4a-35af-a071-46f04d6ea4fa</destinationId>
+ <destinationGroupId>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <processGroup>
+ <id>b997e46b-7905-33e8-8bbc-f4d51b0cc735</id>
+ <name>Tor Nodes</name>
+ <position x="968.7335178760902" y="456.9915202898361" />
+ <comment>Downloads a CSV file of IP addresses used as Tor nodes</comment>
+ <processor>
+ <id>ad366a87-89d6-38ff-affe-a1f3575faa8a</id>
+ <name>Save to disk</name>
+ <position x="-328.58331298828125" y="-153.10000610351562" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.PutFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Directory</name>
+ <value>/opt/nifi/nifi-current/conf/</value>
+ </property>
+ <property>
+ <name>Conflict Resolution Strategy</name>
+ <value>replace</value>
+ </property>
+ <property>
+ <name>Create Missing Directories</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Maximum File Count</name>
+ </property>
+ <property>
+ <name>Last Modified Time</name>
+ </property>
+ <property>
+ <name>Permissions</name>
+ </property>
+ <property>
+ <name>Owner</name>
+ </property>
+ <property>
+ <name>Group</name>
+ </property>
+ <autoTerminatedRelationship>success</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>34f52e1e-164e-34e4-b5fc-e5d16f773b19</id>
+ <name>Get CSV file with Tor nodes</name>
+ <position x="-323.0833282470703" y="-647.6000061035156" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.GetHTTP</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>1 day</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>URL</name>
+ <value>http://check.torproject.org/torbulkexitlist</value>
+ </property>
+ <property>
+ <name>Filename</name>
+ <value>tornodes.csv</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ </property>
+ <property>
+ <name>Username</name>
+ </property>
+ <property>
+ <name>Password</name>
+ </property>
+ <property>
+ <name>Connection Timeout</name>
+ <value>30 sec</value>
+ </property>
+ <property>
+ <name>Data Timeout</name>
+ <value>30 sec</value>
+ </property>
+ <property>
+ <name>User Agent</name>
+ </property>
+ <property>
+ <name>Accept Content-Type</name>
+ </property>
+ <property>
+ <name>Follow Redirects</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>redirect-cookie-policy</name>
+ <value>default</value>
+ </property>
+ <property>
+ <name>proxy-configuration-service</name>
+ </property>
+ <property>
+ <name>Proxy Host</name>
+ </property>
+ <property>
+ <name>Proxy Port</name>
+ </property>
+ </processor>
+ <processor>
+ <id>8c69ccb6-616f-3ce2-b0cd-57276cae3749</id>
+ <name>Add header</name>
+ <position x="-325.5833282470703" y="-410.1000061035156" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ReplaceText</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Regular Expression</name>
+ <value>(?s)(^.*$)</value>
+ </property>
+ <property>
+ <name>Replacement Value</name>
+ <value>ip_addr
+</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Maximum Buffer Size</name>
+ <value>1 MB</value>
+ </property>
+ <property>
+ <name>Replacement Strategy</name>
+ <value>Prepend</value>
+ </property>
+ <property>
+ <name>Evaluation Mode</name>
+ <value>Entire text</value>
+ </property>
+ <property>
+ <name>Line-by-Line Evaluation Mode</name>
+ <value>All</value>
+ </property>
+ </processor>
+ <connection>
+ <id>33cb6d60-d003-3954-b9d0-f51ac40ed983</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</sourceId>
+ <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>ad366a87-89d6-38ff-affe-a1f3575faa8a</destinationId>
+ <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>cd13f619-bb19-37c2-b8fe-c962edfbf213</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="137.4166717529297" y="-370.1000061035156" />
+ <bendPoint x="137.4166717529297" y="-320.1000061035156" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</sourceId>
+ <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</destinationId>
+ <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>3941ee86-e740-3b8f-951a-c7da71e78fbe</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>34f52e1e-164e-34e4-b5fc-e5d16f773b19</sourceId>
+ <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</destinationId>
+ <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>19de0f5c-c244-3e7d-b711-ee165b493ea2</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="134.41668701171875" y="-113.10000610351562" />
+ <bendPoint x="134.41668701171875" y="-63.100006103515625" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>ad366a87-89d6-38ff-affe-a1f3575faa8a</sourceId>
+ <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>ad366a87-89d6-38ff-affe-a1f3575faa8a</destinationId>
+ <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>8130df3d-dc8c-32c2-975d-9c94438cac05</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>34f52e1e-164e-34e4-b5fc-e5d16f773b19</sourceId>
+ <sourceGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>8c69ccb6-616f-3ce2-b0cd-57276cae3749</destinationId>
+ <destinationGroupId>b997e46b-7905-33e8-8bbc-f4d51b0cc735</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <processGroup>
+ <id>194a653f-0c92-3704-8bd4-ffa079643515</id>
+ <name>Misp</name>
+ <position x="548.9658647769079" y="453.4916238226681" />
+ <comment>Polls Misp database once every minute and places new IOCs in a NiFi memcache.</comment>
+ <processor>
+ <id>283bea4b-2774-3f2d-aabe-cf96989e9997</id>
+ <name>Set timestamp as FlowFile content</name>
+ <position x="506.47715414708637" y="587.6551663734834" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.ReplaceText</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Regular Expression</name>
+ <value>(?s)(^.*$)</value>
+ </property>
+ <property>
+ <name>Replacement Value</name>
+ <value>${timestamp}</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>Maximum Buffer Size</name>
+ <value>1 MB</value>
+ </property>
+ <property>
+ <name>Replacement Strategy</name>
+ <value>Always Replace</value>
+ </property>
+ <property>
+ <name>Evaluation Mode</name>
+ <value>Entire text</value>
+ </property>
+ <property>
+ <name>Line-by-Line Evaluation Mode</name>
+ <value>All</value>
+ </property>
+ </processor>
+ <processor>
+ <id>e1e2caef-8178-3c91-b3ca-99f05f619064</id>
+ <name>Get timestamp of last successful poll</name>
+ <position x="-168.51082396716333" y="-293.9956980367642" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.FetchDistributedMapCache</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Cache Entry Identifier</name>
+ <value>${lookup_id}</value>
+ </property>
+ <property>
+ <name>Distributed Cache Service</name>
+ <value>ad4d31bf-b1fb-35e0-b634-b969b200f3a6</value>
+ </property>
+ <property>
+ <name>Put Cache Value In Attribute</name>
+ <value>last_run</value>
+ </property>
+ <property>
+ <name>Max Length To Put In Attribute</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>Character Set</name>
+ <value>UTF-8</value>
+ </property>
+ </processor>
+ <processor>
+ <id>192802be-4416-3abc-ba03-90934f2df860</id>
+ <name>Get events</name>
+ <position x="-151.66592451726592" y="335.6012170464188" />
+ <styles />
+ <comment>Normally the query will have a filter at the end "/last:${last}" so that only new events are pulled. This has been removed from this demo.</comment>
+ <class>org.apache.nifi.processors.standard.InvokeHTTP</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>HTTP Method</name>
+ <value>GET</value>
+ </property>
+ <property>
+ <name>Remote URL</name>
+ <value>${misp_url}/attributes/restSearch/returnFormat:json/type:ip-src||ip-dst</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
+ </property>
+ <property>
+ <name>Connection Timeout</name>
+ <value>5 secs</value>
+ </property>
+ <property>
+ <name>Read Timeout</name>
+ <value>15 secs</value>
+ </property>
+ <property>
+ <name>Include Date Header</name>
+ <value>True</value>
+ </property>
+ <property>
+ <name>Follow Redirects</name>
+ <value>True</value>
+ </property>
+ <property>
+ <name>Attributes to Send</name>
+ </property>
+ <property>
+ <name>Basic Authentication Username</name>
+ </property>
+ <property>
+ <name>Basic Authentication Password</name>
+ </property>
+ <property>
+ <name>proxy-configuration-service</name>
+ </property>
+ <property>
+ <name>Proxy Host</name>
+ </property>
+ <property>
+ <name>Proxy Port</name>
+ </property>
+ <property>
+ <name>Proxy Type</name>
+ <value>http</value>
+ </property>
+ <property>
+ <name>invokehttp-proxy-user</name>
+ </property>
+ <property>
+ <name>invokehttp-proxy-password</name>
+ </property>
+ <property>
+ <name>Put Response Body In Attribute</name>
+ </property>
+ <property>
+ <name>Max Length To Put In Attribute</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>Digest Authentication</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Always Output Response</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Add Response Headers to Request</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Content-Type</name>
+ <value>${mime.type}</value>
+ </property>
+ <property>
+ <name>send-message-body</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Use Chunked Encoding</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Penalize on "No Retry"</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>use-etag</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>etag-max-cache-size</name>
+ <value>10MB</value>
+ </property>
+ <property>
+ <name>ignore-response-content</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Authorization</name>
+ <value>${misp_token}</value>
+ </property>
+ <autoTerminatedRelationship>Original</autoTerminatedRelationship>
+ <autoTerminatedRelationship>Retry</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>671c4e42-604f-389d-9cee-27431ca36448</id>
+ <name>Store timestamp</name>
+ <position x="504.4604101497308" y="824.0677052542044" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.PutDistributedMapCache</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Cache Entry Identifier</name>
+ <value>${lookup_id}</value>
+ </property>
+ <property>
+ <name>Distributed Cache Service</name>
+ <value>ad4d31bf-b1fb-35e0-b634-b969b200f3a6</value>
+ </property>
+ <property>
+ <name>Cache update strategy</name>
+ <value>replace</value>
+ </property>
+ <property>
+ <name>Max cache entry size</name>
+ <value>1 MB</value>
+ </property>
+ <autoTerminatedRelationship>success</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>c7cc4e24-7d9a-3a17-8af1-ca655f46595f</id>
+ <name>Update cache</name>
+ <position x="-775.4735301448745" y="930.3624699197178" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.PutDistributedMapCache</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Cache Entry Identifier</name>
+ <value>${misp_ip}</value>
+ </property>
+ <property>
+ <name>Distributed Cache Service</name>
+ <value>ad4d31bf-b1fb-35e0-b634-b969b200f3a6</value>
+ </property>
+ <property>
+ <name>Cache update strategy</name>
+ <value>replace</value>
+ </property>
+ <property>
+ <name>Max cache entry size</name>
+ <value>1 MB</value>
+ </property>
+ <autoTerminatedRelationship>success</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>81ec71a0-719a-3205-9360-6a535072f7c6</id>
+ <name>Set attributes to get all events for the last x days</name>
+ <position x="-378.1916613806792" y="-12.197472102501479" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>last</name>
+ <value>${misp_ip_first_interval}</value>
+ </property>
+ <property>
+ <name>timestamp</name>
+ <value>${now():toNumber()}</value>
+ </property>
+ </processor>
+ <processor>
+ <id>6d78b76c-5463-3610-b8c8-4796fa09c59b</id>
+ <name>Periodic polling</name>
+ <position x="-171.36520083798905" y="-518.6967632987289" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>1 minute</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>File Size</name>
+ <value>0B</value>
+ </property>
+ <property>
+ <name>Batch Size</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>Data Format</name>
+ <value>Text</value>
+ </property>
+ <property>
+ <name>Unique FlowFiles</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>generate-ff-custom-text</name>
+ </property>
+ <property>
+ <name>character-set</name>
+ <value>UTF-8</value>
+ </property>
+ <property>
+ <name>lookup_id</name>
+ <value>ip</value>
+ </property>
+ </processor>
+ <processor>
+ <id>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</id>
+ <name>Extract IP address</name>
+ <position x="-156.69110558236184" y="543.7042207790005" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.EvaluateJsonPath</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Destination</name>
+ <value>flowfile-attribute</value>
+ </property>
+ <property>
+ <name>Return Type</name>
+ <value>auto-detect</value>
+ </property>
+ <property>
+ <name>Path Not Found Behavior</name>
+ <value>ignore</value>
+ </property>
+ <property>
+ <name>Null Value Representation</name>
+ <value>empty string</value>
+ </property>
+ <property>
+ <name>misp_ip</name>
+ <value>$.value</value>
+ </property>
+ </processor>
+ <processor>
+ <id>74d66e0e-0b65-36d2-96f1-4b836d2c4222</id>
+ <name>Set attributes to get new events since last poll</name>
+ <position x="81.93877074822706" y="-13.058372981407729" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.attributes.UpdateAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-update-attribute-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Delete Attributes Expression</name>
+ </property>
+ <property>
+ <name>Store State</name>
+ <value>Do not store state</value>
+ </property>
+ <property>
+ <name>Stateful Variables Initial Value</name>
+ </property>
+ <property>
+ <name>canonical-value-lookup-cache-size</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>last</name>
+ <value>${now():toNumber():minus(${last_run}):divide(60000):plus(1):append("m")}</value>
+ </property>
+ <property>
+ <name>timestamp</name>
+ <value>${now():toNumber()}</value>
+ </property>
+ </processor>
+ <processor>
+ <id>ba1b7e7e-a03c-3ace-9182-7f43569537e2</id>
+ <name>Create one FlowFile for each IP address</name>
+ <position x="-789.5267777615984" y="546.1428879861119" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.SplitJson</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>JsonPath Expression</name>
+ <value>$.response.Attribute</value>
+ </property>
+ <property>
+ <name>Null Value Representation</name>
+ <value>empty string</value>
+ </property>
+ <autoTerminatedRelationship>original</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>d850fc04-df9a-36b7-b53f-8b397a1be69a</id>
+ <name>Extract Misp event ID and store it to FlowFile</name>
+ <position x="-783.5607955237681" y="719.2550630641567" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.EvaluateJsonPath</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Destination</name>
+ <value>flowfile-content</value>
+ </property>
+ <property>
+ <name>Return Type</name>
+ <value>auto-detect</value>
+ </property>
+ <property>
+ <name>Path Not Found Behavior</name>
+ <value>ignore</value>
+ </property>
+ <property>
+ <name>Null Value Representation</name>
+ <value>empty string</value>
+ </property>
+ <property>
+ <name>event_id</name>
+ <value>$.event_id</value>
+ </property>
+ </processor>
+ <funnel>
+ <id>c490b6b5-0170-1000-0000-000035bc685d</id>
+ <position x="601.9534533822577" y="371.9240905653907" />
+ </funnel>
+ <connection>
+ <id>39f7b787-0995-3721-8d50-700838b7a256</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>6d78b76c-5463-3610-b8c8-4796fa09c59b</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>e1e2caef-8178-3c91-b3ca-99f05f619064</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>adc3f55b-8d9a-33d0-a7af-0d795fa234ba</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>d850fc04-df9a-36b7-b53f-8b397a1be69a</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c7cc4e24-7d9a-3a17-8af1-ca655f46595f</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>matched</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>4a797ab8-fb0e-3c9a-b397-b3394eca1ce4</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>ba1b7e7e-a03c-3ace-9182-7f43569537e2</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>split</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>4707ebf2-4b30-3e97-8abc-6ca8a9d168fd</id>
+ <name>Consecutive poll</name>
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e1e2caef-8178-3c91-b3ca-99f05f619064</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>74d66e0e-0b65-36d2-96f1-4b836d2c4222</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>58cc41df-404e-309b-9df6-2ea67e1fe2b7</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>283bea4b-2774-3f2d-aabe-cf96989e9997</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>671c4e42-604f-389d-9cee-27431ca36448</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5e84f4ae-bf61-37d8-b115-0af74b89a6aa</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>d850fc04-df9a-36b7-b53f-8b397a1be69a</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>matched</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>a9d8c7a2-6b55-3684-9954-92934d5a69e8</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>74d66e0e-0b65-36d2-96f1-4b836d2c4222</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>192802be-4416-3abc-ba03-90934f2df860</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c2a99429-58ab-325b-b755-dffeb30b0fc1</id>
+ <name />
+ <bendPoints />
+ <labelIndex>0</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>192802be-4416-3abc-ba03-90934f2df860</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c490b6b5-0170-1000-0000-000035bc685d</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>No Retry</relationship>
+ <relationship>Failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>a146ab21-f626-3fa5-a736-fdeec786eaf8</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-312.47353014487453" y="970.3624699197178" />
+ <bendPoint x="-312.47353014487453" y="1020.3624699197178" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c7cc4e24-7d9a-3a17-8af1-ca655f46595f</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c7cc4e24-7d9a-3a17-8af1-ca655f46595f</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c022992b-534a-317a-943c-86142ee1cf81</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>81ec71a0-719a-3205-9360-6a535072f7c6</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>192802be-4416-3abc-ba03-90934f2df860</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>424c08f5-1ad1-3a0a-923c-c3fd988f7d2e</id>
+ <name>Update timestamp</name>
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>192802be-4416-3abc-ba03-90934f2df860</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>283bea4b-2774-3f2d-aabe-cf96989e9997</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>Response</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>5b9b78c7-890c-3fe0-a1b1-b2dc5bbd944b</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="306.30889441763816" y="583.7042207790005" />
+ <bendPoint x="306.30889441763816" y="633.7042207790005" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>dbc236e3-8c68-3c6b-b1e9-d1fc8f57327d</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <relationship>unmatched</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c98425f5-d4f3-36f7-b045-834923ca235a</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-911.4283280545671" y="539.9664353493931" />
+ <bendPoint x="-945.2909989530046" y="613.0546921853306" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>ba1b7e7e-a03c-3ace-9182-7f43569537e2</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>ba1b7e7e-a03c-3ace-9182-7f43569537e2</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>d19c9d34-8896-39ff-9d2d-f29651b24c18</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="967.4604101497307" y="864.0677052542044" />
+ <bendPoint x="967.4604101497307" y="914.0677052542044" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>671c4e42-604f-389d-9cee-27431ca36448</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>671c4e42-604f-389d-9cee-27431ca36448</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>70fec2c7-4dd5-3dd1-92fa-59c3027bffb4</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="294.48917603283667" y="-253.99569803676422" />
+ <bendPoint x="294.48917603283667" y="-203.99569803676422" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e1e2caef-8178-3c91-b3ca-99f05f619064</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>e1e2caef-8178-3c91-b3ca-99f05f619064</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>d19116d2-0da0-3f86-8fd3-3285a839648e</id>
+ <name>First poll</name>
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>e1e2caef-8178-3c91-b3ca-99f05f619064</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>81ec71a0-719a-3205-9360-6a535072f7c6</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>not-found</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>d375a69b-2139-3d9a-b6e3-48e0f69ec589</id>
+ <name>Update cache with new events</name>
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>192802be-4416-3abc-ba03-90934f2df860</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>ba1b7e7e-a03c-3ace-9182-7f43569537e2</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>Response</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>8d3dfbef-370e-374b-a6c6-89e4cdf6216b</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="-320.5607955237681" y="759.2550630641567" />
+ <bendPoint x="-320.5607955237681" y="809.2550630641567" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>d850fc04-df9a-36b7-b53f-8b397a1be69a</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>d850fc04-df9a-36b7-b53f-8b397a1be69a</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <relationship>unmatched</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>536817e5-12a1-3a94-82ae-7638937a07e8</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="969.4771541470864" y="627.6551663734834" />
+ <bendPoint x="969.4771541470864" y="677.6551663734834" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>283bea4b-2774-3f2d-aabe-cf96989e9997</sourceId>
+ <sourceGroupId>194a653f-0c92-3704-8bd4-ffa079643515</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>283bea4b-2774-3f2d-aabe-cf96989e9997</destinationId>
+ <destinationGroupId>194a653f-0c92-3704-8bd4-ffa079643515</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ <processGroup>
+ <id>c4a200ea-5317-332a-97a4-ff76f951ecde</id>
+ <name>GeoIP</name>
+ <position x="556.427978515625" y="673.0274658203125" />
+ <comment />
+ <processor>
+ <id>6292665b-f188-3551-b366-95476b5ac36f</id>
+ <name>Save to disk</name>
+ <position x="-357.78594755036767" y="656.471512008819" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.PutFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Directory</name>
+ <value>/opt/nifi/nifi-current/conf/</value>
+ </property>
+ <property>
+ <name>Conflict Resolution Strategy</name>
+ <value>replace</value>
+ </property>
+ <property>
+ <name>Create Missing Directories</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Maximum File Count</name>
+ </property>
+ <property>
+ <name>Last Modified Time</name>
+ </property>
+ <property>
+ <name>Permissions</name>
+ </property>
+ <property>
+ <name>Owner</name>
+ </property>
+ <property>
+ <name>Group</name>
+ </property>
+ <autoTerminatedRelationship>success</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>c8b26516-0170-1000-ffff-fffffa357a77</id>
+ <name>InvokeHTTP</name>
+ <position x="-354.33263208075834" y="-1.6134650355261897" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.InvokeHTTP</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>HTTP Method</name>
+ <value>GET</value>
+ </property>
+ <property>
+ <name>Remote URL</name>
+ <value>https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${maxmind_key}&suffix=tar.gz</value>
+ </property>
+ <property>
+ <name>SSL Context Service</name>
+ <value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
+ </property>
+ <property>
+ <name>Connection Timeout</name>
+ <value>5 secs</value>
+ </property>
+ <property>
+ <name>Read Timeout</name>
+ <value>15 secs</value>
+ </property>
+ <property>
+ <name>Include Date Header</name>
+ <value>True</value>
+ </property>
+ <property>
+ <name>Follow Redirects</name>
+ <value>True</value>
+ </property>
+ <property>
+ <name>Attributes to Send</name>
+ </property>
+ <property>
+ <name>Basic Authentication Username</name>
+ </property>
+ <property>
+ <name>Basic Authentication Password</name>
+ </property>
+ <property>
+ <name>proxy-configuration-service</name>
+ </property>
+ <property>
+ <name>Proxy Host</name>
+ </property>
+ <property>
+ <name>Proxy Port</name>
+ </property>
+ <property>
+ <name>Proxy Type</name>
+ <value>http</value>
+ </property>
+ <property>
+ <name>invokehttp-proxy-user</name>
+ </property>
+ <property>
+ <name>invokehttp-proxy-password</name>
+ </property>
+ <property>
+ <name>Put Response Body In Attribute</name>
+ </property>
+ <property>
+ <name>Max Length To Put In Attribute</name>
+ <value>256</value>
+ </property>
+ <property>
+ <name>Digest Authentication</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Always Output Response</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Add Response Headers to Request</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Content-Type</name>
+ <value>${mime.type}</value>
+ </property>
+ <property>
+ <name>send-message-body</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>Use Chunked Encoding</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>Penalize on "No Retry"</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>use-etag</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>etag-max-cache-size</name>
+ <value>10MB</value>
+ </property>
+ <property>
+ <name>ignore-response-content</name>
+ <value>false</value>
+ </property>
+ <autoTerminatedRelationship>Original</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>b99eab15-7e38-33fa-87d1-41d772306d9c</id>
+ <name>Uncompress</name>
+ <position x="-359.13545011384423" y="239.87525101326742" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.CompressContent</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Mode</name>
+ <value>decompress</value>
+ </property>
+ <property>
+ <name>Compression Format</name>
+ <value>use mime.type attribute</value>
+ </property>
+ <property>
+ <name>Compression Level</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>Update Filename</name>
+ <value>false</value>
+ </property>
+ </processor>
+ <processor>
+ <id>c8b20333-0170-1000-0000-000010760524</id>
+ <name>RouteOnAttribute</name>
+ <position x="-353.2358571852152" y="-223.16639543708658" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.RouteOnAttribute</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Routing Strategy</name>
+ <value>Route to Property name</value>
+ </property>
+ <property>
+ <name>maxmind_key</name>
+ <value>${maxmind_key:length():gt(1)}</value>
+ </property>
+ <autoTerminatedRelationship>unmatched</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>aad91df7-8e80-3598-a3eb-9b000045b843</id>
+ <name>UnpackContent</name>
+ <position x="-358.13545011384423" y="448.82544805040084" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.UnpackContent</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>0 sec</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>RUNNING</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>Packaging Format</name>
+ <value>tar</value>
+ </property>
+ <property>
+ <name>File Filter</name>
+ <value>GeoLite2-City.mmdb</value>
+ </property>
+ <autoTerminatedRelationship>original</autoTerminatedRelationship>
+ </processor>
+ <processor>
+ <id>c8b1bafd-0170-1000-0000-0000753f5f5b</id>
+ <name>GenerateFlowFile</name>
+ <position x="-366.3974570271698" y="-455.687252544095" />
+ <styles />
+ <comment />
+ <class>org.apache.nifi.processors.standard.GenerateFlowFile</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-standard-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <maxConcurrentTasks>1</maxConcurrentTasks>
+ <schedulingPeriod>1 week</schedulingPeriod>
+ <penalizationPeriod>30 sec</penalizationPeriod>
+ <yieldPeriod>1 sec</yieldPeriod>
+ <bulletinLevel>WARN</bulletinLevel>
+ <lossTolerant>false</lossTolerant>
+ <scheduledState>STOPPED</scheduledState>
+ <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
+ <executionNode>ALL</executionNode>
+ <runDurationNanos>0</runDurationNanos>
+ <property>
+ <name>File Size</name>
+ <value>0B</value>
+ </property>
+ <property>
+ <name>Batch Size</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>Data Format</name>
+ <value>Text</value>
+ </property>
+ <property>
+ <name>Unique FlowFiles</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>generate-ff-custom-text</name>
+ </property>
+ <property>
+ <name>character-set</name>
+ <value>UTF-8</value>
+ </property>
+ </processor>
+ <funnel>
+ <id>c2cac6f3-c926-3038-b685-68f71f76fda3</id>
+ <position x="457.8712158203125" y="380.06201171875" />
+ </funnel>
+ <connection>
+ <id>c8b21bba-0170-1000-0000-0000281b44ba</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c8b1bafd-0170-1000-0000-0000753f5f5b</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c8b20333-0170-1000-0000-000010760524</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>1d0b1e6f-7b01-34c5-82f8-c95918e700ae</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>b99eab15-7e38-33fa-87d1-41d772306d9c</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>e8e86b3f-6936-3080-8eb6-036d532cb483</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>6292665b-f188-3551-b366-95476b5ac36f</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>e2f43878-959f-379c-b898-6d7c3a72af44</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>b99eab15-7e38-33fa-87d1-41d772306d9c</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>aad91df7-8e80-3598-a3eb-9b000045b843</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>dc7524de-fb6c-3e02-8c60-f821d81aff29</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>aad91df7-8e80-3598-a3eb-9b000045b843</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>6292665b-f188-3551-b366-95476b5ac36f</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>success</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>39a11e18-3397-3f1a-a020-49b895ff6f81</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>aad91df7-8e80-3598-a3eb-9b000045b843</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c8b2e58b-0170-1000-ffff-ffff997c6e6c</id>
+ <name />
+ <bendPoints>
+ <bendPoint x="108.66736791924166" y="38.38653496447381" />
+ <bendPoint x="108.66736791924166" y="88.38653496447381" />
+ </bendPoints>
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c8b26516-0170-1000-ffff-fffffa357a77</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c8b26516-0170-1000-ffff-fffffa357a77</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>Retry</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c8b29bee-0170-1000-ffff-fffff516df5d</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c8b26516-0170-1000-ffff-fffffa357a77</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>b99eab15-7e38-33fa-87d1-41d772306d9c</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>Response</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c8b2cb01-0170-1000-0000-000005baadda</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c8b26516-0170-1000-ffff-fffffa357a77</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c2cac6f3-c926-3038-b685-68f71f76fda3</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>FUNNEL</destinationType>
+ <relationship>No Retry</relationship>
+ <relationship>Failure</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ <connection>
+ <id>c8b5c90e-0170-1000-ffff-ffff9864e7e4</id>
+ <name />
+ <bendPoints />
+ <labelIndex>1</labelIndex>
+ <zIndex>0</zIndex>
+ <sourceId>c8b20333-0170-1000-0000-000010760524</sourceId>
+ <sourceGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</sourceGroupId>
+ <sourceType>PROCESSOR</sourceType>
+ <destinationId>c8b26516-0170-1000-ffff-fffffa357a77</destinationId>
+ <destinationGroupId>c4a200ea-5317-332a-97a4-ff76f951ecde</destinationGroupId>
+ <destinationType>PROCESSOR</destinationType>
+ <relationship>maxmind_key</relationship>
+ <maxWorkQueueSize>10000</maxWorkQueueSize>
+ <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
+ <flowFileExpiration>0 sec</flowFileExpiration>
+ <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
+ <partitioningAttribute />
+ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
+ </connection>
+ </processGroup>
+ </processGroup>
<controllerService>
- <id>bc97858d-0175-1000-0000-0000130a84f8</id>
- <name>Nifi logs GrokReader</name>
+ <id>bf81debc-0171-1000-0000-00002936ae5a</id>
+ <name>Tor node CSV</name>
<comment />
- <class>org.apache.nifi.grok.GrokReader</class>
+ <class>org.apache.nifi.lookup.CSVRecordLookupService</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-lookup-services-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
- <name>schema-access-strategy</name>
- <value>string-fields-from-grok-expression</value>
- </property>
- <property>
- <name>schema-registry</name>
- </property>
- <property>
- <name>schema-name</name>
- <value>${schema.name}</value>
- </property>
- <property>
- <name>schema-version</name>
- </property>
- <property>
- <name>schema-branch</name>
- </property>
- <property>
- <name>schema-text</name>
- <value>${avro.schema}</value>
- </property>
- <property>
- <name>Grok Pattern File</name>
+ <name>csv-file</name>
+ <value>/opt/nifi/nifi-current/conf/enrich/tornodes.csv</value>
</property>
<property>
- <name>Grok Expression</name>
- <value>%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:class} %{GREEDYDATA:message}</value>
+ <name>csv-format</name>
+ <value>Default</value>
</property>
<property>
- <name>no-match-behavior</name>
- <value>append-to-previous-message</value>
+ <name>Character Set</name>
+ <value>UTF-8</value>
</property>
- </controllerService>
- <controllerService>
- <id>bc8e5957-0175-1000-0000-00003346421d</id>
- <name>Extract message field</name>
- <comment />
- <class>org.apache.nifi.text.FreeFormTextRecordSetWriter</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
<property>
- <name>Text</name>
- <value>${message}</value>
+ <name>lookup-key-column</name>
+ <value>ip_addr</value>
</property>
<property>
- <name>Character Set</name>
- <value>UTF-8</value>
+ <name>ignore-duplicates</name>
+ <value>true</value>
</property>
</controllerService>
<controllerService>
@@ -11503,7 +8591,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-lookup-services-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
@@ -11523,7 +8611,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-distributed-cache-services-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
@@ -11542,46 +8630,6 @@
<value>30 secs</value>
</property>
</controllerService>
- <controllerService>
- <id>83443c00-b286-366a-b8e0-2f51527ab8e5</id>
- <name>Soctools CA</name>
- <comment />
- <class>org.apache.nifi.ssl.StandardRestrictedSSLContextService</class>
- <bundle>
- <group>org.apache.nifi</group>
- <artifact>nifi-ssl-context-service-nar</artifact>
- <version>1.12.1</version>
- </bundle>
- <enabled>true</enabled>
- <property>
- <name>Keystore Filename</name>
- </property>
- <property>
- <name>Keystore Password</name>
- </property>
- <property>
- <name>key-password</name>
- </property>
- <property>
- <name>Keystore Type</name>
- </property>
- <property>
- <name>Truststore Filename</name>
- <value>/opt/nifi/nifi-current/conf/cacerts.jks</value>
- </property>
- <property>
- <name>Truststore Password</name>
- <value>{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}</value>
- </property>
- <property>
- <name>Truststore Type</name>
- <value>JKS</value>
- </property>
- <property>
- <name>SSL Protocol</name>
- <value>TLS</value>
- </property>
- </controllerService>
<controllerService>
<id>17b30955-5464-3709-8a32-69a459850cfa</id>
<name>Inferred JsonRecordSetWriter</name>
@@ -11590,7 +8638,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
@@ -11600,10 +8648,6 @@
<property>
<name>schema-cache</name>
</property>
- <property>
- <name>schema-protocol-version</name>
- <value>1</value>
- </property>
<property>
<name>schema-access-strategy</name>
<value>inherit-record-schema</value>
@@ -11656,47 +8700,74 @@
</property>
</controllerService>
<controllerService>
- <id>70ea12d7-0176-1000-ffff-ffffee2ee306</id>
- <name>Mysql log GrokReader</name>
+ <id>8b1dd8bb-0170-1000-0000-000007446e6a</id>
+ <name>Misp DistributedMapCacheServer</name>
<comment />
- <class>org.apache.nifi.grok.GrokReader</class>
+ <class>org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer</class>
<bundle>
<group>org.apache.nifi</group>
- <artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
+ <artifact>nifi-distributed-cache-services-nar</artifact>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
- <name>schema-access-strategy</name>
- <value>string-fields-from-grok-expression</value>
+ <name>Port</name>
+ <value>4557</value>
</property>
<property>
- <name>schema-registry</name>
+ <name>Maximum Cache Entries</name>
+ <value>10000</value>
</property>
<property>
- <name>schema-name</name>
- <value>${schema.name}</value>
+ <name>Eviction Strategy</name>
+ <value>Least Frequently Used</value>
</property>
<property>
- <name>schema-version</name>
+ <name>Persistence Directory</name>
+ <value>/opt/nifi/nifi-current/conf/</value>
</property>
<property>
- <name>schema-branch</name>
+ <name>SSL Context Service</name>
</property>
+ </controllerService>
+ <controllerService>
+ <id>83443c00-b286-366a-b8e0-2f51527ab8e5</id>
+ <name>Common CA</name>
+ <comment />
+ <class>org.apache.nifi.ssl.StandardRestrictedSSLContextService</class>
+ <bundle>
+ <group>org.apache.nifi</group>
+ <artifact>nifi-ssl-context-service-nar</artifact>
+ <version>1.11.4</version>
+ </bundle>
+ <enabled>true</enabled>
<property>
- <name>schema-text</name>
- <value>${avro.schema}</value>
+ <name>Keystore Filename</name>
+ </property>
+ <property>
+ <name>Keystore Password</name>
+ </property>
+ <property>
+ <name>key-password</name>
+ </property>
+ <property>
+ <name>Keystore Type</name>
+ </property>
+ <property>
+ <name>Truststore Filename</name>
+ <value>/opt/nifi/nifi-current/conf/cacerts.jks</value>
</property>
<property>
- <name>Grok Pattern File</name>
+ <name>Truststore Password</name>
+ <value>enc{a4ca3924cb58cb8c28fec2766ce1a66f9bec9ca13f5cb90008f3b0719d4777b2}</value>
</property>
<property>
- <name>Grok Expression</name>
- <value>%{GREEDYDATA:timestamp} %{DATA:process}: %{GREEDYDATA:message}</value>
+ <name>Truststore Type</name>
+ <value>JKS</value>
</property>
<property>
- <name>no-match-behavior</name>
- <value>append-to-previous-message</value>
+ <name>SSL Protocol</name>
+ <value>TLS</value>
</property>
</controllerService>
<controllerService>
@@ -11707,7 +8778,7 @@
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-record-serialization-services-nar</artifact>
- <version>1.12.1</version>
+ <version>1.11.4</version>
</bundle>
<enabled>true</enabled>
<property>
@@ -11744,13 +8815,13 @@
<name>Timestamp Format</name>
</property>
</controllerService>
- <variable name="misp_token" value="{{lookup('file','{{playbook_dir}}/secrets/tokens/misp')}}" />
+ <variable name="misp_token" value="{{ misp_token }}" />
<variable name="maxmind_key" value="{{ maxmind_key }}" />
- <variable name="misp_first_interval" value="60d" />
+ <variable name="misp_ip_first_interval" value="60d" />
<variable name="elastic_username" value="{{ elastic_username }}" />
<variable name="misp_url" value="{{ misp_url }}" />
- <variable name="elastic_url" value="https://{{ soctoolsproxy }}:9200" />
- <variable name="elastic_password" value="{{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}" />
+ <variable name="elastic_url" value="https://{{ dslproxy }}:9200" />
+ <variable name="elastic_password" value="{{ odfees_adminpass }}" />
</rootGroup>
<controllerServices />
<reportingTasks />
diff --git a/roles/nifi/templates/nifi.properties.j2 b/roles/nifi/templates/nifi.properties.j2
index c2dafaff8e4019ebc0e5c6a206b3378863110f80..426e5ce0d6c5975ff27cef19449d2deea9a93b20 100644
--- a/roles/nifi/templates/nifi.properties.j2
+++ b/roles/nifi/templates/nifi.properties.j2
@@ -120,8 +120,8 @@ nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
-nifi.components.status.repository.buffer.size=288
-nifi.components.status.snapshot.frequency=5 min
+nifi.components.status.repository.buffer.size=1440
+nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host={{ inventory_hostname }}
@@ -143,7 +143,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
nifi.web.max.header.size=16 KB
nifi.web.proxy.context.path=/nifi
-nifi.web.proxy.host={{ soctoolsproxy }}:9443
+nifi.web.proxy.host={{ dslproxy }}:9443
# security properties #
nifi.sensitive.props.key=
@@ -154,21 +154,21 @@ nifi.sensitive.props.additional.keys=
nifi.security.keystore=./conf/{{ inventory_hostname }}.p12
nifi.security.keystoreType=pkcs12
-nifi.security.keystorePasswd={{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
+nifi.security.keystorePasswd={{ kspass}}
#nifi.security.keyPasswd=IP7Jgn7amiAYi3LRSRk5LGg3t4zlfh0kEKcAaaoxHDo
nifi.security.truststore=./conf/cacerts.jks
nifi.security.truststoreType=jks
-nifi.security.truststorePasswd={{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}
+nifi.security.truststorePasswd={{ tspass}}
nifi.security.user.authorizer=managed-authorizer
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# OpenId Connect SSO Properties #
-nifi.security.user.oidc.discovery.url=https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
-nifi.security.user.oidc.connect.timeout=10 secs
-nifi.security.user.oidc.read.timeout=10 secs
-nifi.security.user.oidc.client.id=soctools-nifi
+nifi.security.user.oidc.discovery.url=https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
+nifi.security.user.oidc.connect.timeout=5 secs
+nifi.security.user.oidc.read.timeout=5 secs
+nifi.security.user.oidc.client.id=dsoclab-nifi
nifi.security.user.oidc.client.secret={{nifisecret.value}}
nifi.security.user.oidc.preferred.jwsalgorithm=
nifi.security.user.oidc.additional.scopes={{openid_scope}}
@@ -226,9 +226,9 @@ nifi.cluster.load.balance.max.thread.count=8
nifi.cluster.load.balance.comms.timeout=30 sec
# zookeeper properties, used for cluster management #
-nifi.zookeeper.connect.string=soctools-zookeeper:2181
-nifi.zookeeper.connect.timeout=60 secs
-nifi.zookeeper.session.timeout=60 secs
+nifi.zookeeper.connect.string=dsoclab-zookeeper:2181
+nifi.zookeeper.connect.timeout=3 secs
+nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management
diff --git a/roles/odfees/tasks/main.yml b/roles/odfees/tasks/main.yml
index 016e8786ed6a31238dd2c7efc68e9aec7c7a9912..ae6ae65701c4bebcf8cbd04bc4d4ddb9e246acfd 100644
--- a/roles/odfees/tasks/main.yml
+++ b/roles/odfees/tasks/main.yml
@@ -1,17 +1,118 @@
---
-- include: start.yml
+- name: Copy cacert to ca-trust dir
+ remote_user: root
+ copy:
+ src: "files/{{ca_cn}}.crt"
+ dest: /etc/pki/ca-trust/source/anchors/ca.crt
tags:
- - start
-- include: stop.yml
+ - start
+
+- name: Install cacert to root truststore
+ remote_user: root
+ command: "update-ca-trust"
+ tags:
+ - start
+
+- name: Copy certificates in odfe conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "config/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.p12"
+ - cacerts.jks
+ - "{{soctools_users[0].CN}}.p12"
+ tags:
+ - start
+
+- name: Configure sysconfig
+ template:
+ src: sysconfig_elasticsearch.j2
+ dest: sysconfig_elasticsearch
+ tags:
+ - start
+
+- name: Copy sysconfig to /etc
+ command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
+ tags:
+ - start
+
+- name: Configure odfe properties
+ template:
+ src: "config/{{item}}.j2"
+ dest: "config/{{item}}"
+ with_items:
+ - elasticsearch.yml
+ - jvm.options
+ - log4j2.properties
+ tags:
+ - start
+
+- name: Change password for admin
+ command: "bash plugins/opendistro_security/tools/hash.sh -p {{odfees_adminpass}}"
+ register: adminhash
+ # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
+ tags:
+ - start
+
+- set_fact:
+ adminhashpwd: "{{ adminhash.stdout }}"
+ #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}"
+ tags:
+ - start
+
+- name: Change password for cortex
+ command: "bash plugins/opendistro_security/tools/hash.sh -p {{cortex_odfe_pass}}"
+ register: cortexhash
+ # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
tags:
- - stop
- - stop-odfees
-- include: update-config.yml
+ - start
+
+- set_fact:
+ cortexhashpwd: "{{ cortexhash.stdout }}"
+ #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}"
tags:
- - update-config
- - update-odfees-config
-- include: restart.yml
+ - start
+
+- name: Configure opendistro_security properties
+ template:
+ src: "securityconfig/{{item}}.j2"
+ dest: "plugins/opendistro_security/securityconfig/{{item}}"
+ with_items:
+ - internal_users.yml
+ - config.yml
+ - roles_mapping.yml
tags:
- - restart
- - restart-odfees
+ - start
+
+#- name: Exit here to test ODFE
+# meta: end_play
+# tags:
+# - start
+
+- name: Start OpenDistro for Elasticsearch
+ command: "/usr/share/elasticsearch/bin/elasticsearch -p {{ inventory_hostname }}.pid -d"
+ tags:
+ - start
+
+- name: Wait for ElasticSearch
+ wait_for:
+ host: "{{groups['odfeescontainers'][0]}}"
+ port: 9200
+ state: started
+ delay: 5
+ tags:
+ - start
+
+- name: Configure OpenDistro security
+ command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{soctools_users[0].CN}}.p12' -kspass {{soctools_users[0].password}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{tspass}} -cn dsoclab-cluster"
+ when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
+ tags:
+ - start
+
+- name: Stop OpenDistro for Elasticsearch
+ command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+ tags:
+ - stop
+
diff --git a/roles/odfees/templates/config/elasticsearch.yml.j2 b/roles/odfees/templates/config/elasticsearch.yml.j2
index 5e8e18fc2999f2622cca3b0c229265a379c49b44..ef61cd36dac6bdefebeaeab66a3c15b97aa3a25d 100644
--- a/roles/odfees/templates/config/elasticsearch.yml.j2
+++ b/roles/odfees/templates/config/elasticsearch.yml.j2
@@ -1,4 +1,4 @@
-cluster.name: "soctools-cluster"
+cluster.name: "dsoclab-cluster"
#network.host: 0.0.0.0
network.host: {{ inventory_hostname }}
discovery.seed_hosts:
@@ -30,11 +30,11 @@ cluster.initial_master_nodes:
opendistro_security.ssl.transport.keystore_type: pkcs12
opendistro_security.ssl.transport.keystore_filepath: {{ inventory_hostname }}.p12
-opendistro_security.ssl.transport.keystore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
+opendistro_security.ssl.transport.keystore_password: {{ kspass }}
#opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
opendistro_security.ssl.transport.truststore_type: jks
opendistro_security.ssl.transport.truststore_filepath: cacerts.jks
-opendistro_security.ssl.transport.truststore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
+opendistro_security.ssl.transport.truststore_password: {{ tspass }}
opendistro_security.ssl.transport.enforce_hostname_verification: false
opendistro_security.ssl.http.enabled: true
@@ -42,10 +42,10 @@ opendistro_security.ssl.http.enabled: true
# opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem
opendistro_security.ssl.http.keystore_type: pkcs12
opendistro_security.ssl.http.keystore_filepath: {{ inventory_hostname }}.p12
-opendistro_security.ssl.http.keystore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
+opendistro_security.ssl.http.keystore_password: {{ kspass }}
opendistro_security.ssl.http.truststore_type: jks
opendistro_security.ssl.http.truststore_filepath: cacerts.jks
-opendistro_security.ssl.http.truststore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
+opendistro_security.ssl.http.truststore_password: {{ tspass }}
#opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem
#opendistro_security.ssl.http.clientauth_mode: optional
opendistro_security.allow_unsafe_democertificates: false
diff --git a/roles/odfees/templates/config/log4j2.properties.j2 b/roles/odfees/templates/config/log4j2.properties.j2
index ee01d9a1406720d46fe983efacf16cc8d52c3729..9ad290ad82679309319cee88bee3eaf9d49814eb 100644
--- a/roles/odfees/templates/config/log4j2.properties.j2
+++ b/roles/odfees/templates/config/log4j2.properties.j2
@@ -5,27 +5,5 @@ appender.console.name = console
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
-appender.rolling.type = RollingFile
-appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json
-appender.rolling.layout.type = ESJsonLayout
-appender.rolling.layout.type_name = server
-appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz
-appender.rolling.policies.type = Policies
-appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.rolling.policies.time.interval = 1
-appender.rolling.policies.time.modulate = true
-appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.rolling.policies.size.size = 20MB
-appender.rolling.strategy.type = DefaultRolloverStrategy
-appender.rolling.strategy.fileIndex = nomax
-appender.rolling.strategy.action.type = Delete
-appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
-appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
-appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
-appender.rolling.strategy.action.condition.nested_condition.exceeds = 100MB
-
rootLogger.level = info
-#rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.rolling.ref = rolling
+rootLogger.appenderRef.console.ref = console
diff --git a/roles/odfees/templates/securityconfig/config.yml.j2 b/roles/odfees/templates/securityconfig/config.yml.j2
index 49368676333bb6153b32e988dcd9bd60764426b2..26e77a4fa806fe68767015fc40ad620117985ac5 100644
--- a/roles/odfees/templates/securityconfig/config.yml.j2
+++ b/roles/odfees/templates/securityconfig/config.yml.j2
@@ -116,7 +116,7 @@ config:
config:
subject_key: {{openid_subjkey}}
roles_key: roles
- openid_connect_url: https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
+ openid_connect_url: https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
enable_ssl: true
verify_hostnames: false
pemtrustedcas_filepath: "/usr/share/elasticsearch/config/{{ca_cn}}.crt"
diff --git a/roles/odfekibana/files/kibana_graphs.ndjson b/roles/odfekibana/files/kibana_graphs.ndjson
index f6e604f541a023d488df4f3f76e97d4342d72476..086e784c413a2ff1ad657f886be9030d158a6735 100644
--- a/roles/odfekibana/files/kibana_graphs.ndjson
+++ b/roles/odfekibana/files/kibana_graphs.ndjson
@@ -1,62 +1,11 @@
-{"attributes":{"buildNum":26506,"defaultIndex":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b"},"id":"7.4.2","references":[],"type":"config","updated_at":"2020-12-20T14:02:51.577Z","version":"WzM0LDRd"}
-{"attributes":{"fields":"[{\"name\":\"TLP\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"TLP.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"TLP\",\"subType\":\"multi\"},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"destination.ip\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination.ip.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination.ip\",\"subType\":\"multi\"},{\"name\":\"destination.port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"destination/ip_geo_city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_geo_city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_geo_city\",\"subType\":\"multi\"},{\"name\":\"destination/ip_geo_country\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_geo_country.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_geo_country\",\"subType\":\"multi\"},{\"name\":\"destination/ip_geo_country_iso\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_geo_country_iso.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_geo_country_iso\",\"subType\":\"multi\"},{\"name\":\"destination/ip_geo_lat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_geo_lat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_geo_lat\",\"subType\":\"multi\"},{\"name\":\"destination/ip_geo_lon\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_geo_lon.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_geo_lon\",\"subType\":\"multi\"},{\"name\":\"destination/ip_ipreg_comment\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_ipreg_comment.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_ipreg_comment\",\"subType\":\"multi\"},{\"name\":\"destination/ip_ipreg_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_ipreg_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_ipreg_domain\",\"subType\":\"multi\"},{\"name\":\"destination/ip_ipreg_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"destination/ip_ipreg_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"destination/ip_ipreg_name\",\"subType\":\"multi\"},{\"name\":\"event_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"event_type\",\"subType\":\"multi\"},{\"name\":\"flow_id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_iface\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"in_iface.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"in_iface\",\"subType\":\"multi\"},{\"name\":\"proto\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"proto.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"proto\",\"subType\":\"multi\"},{\"name\":\"source.ip\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source.ip.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source.ip\",\"subType\":\"multi\"},{\"name\":\"source.port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source/ip_geo_city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_geo_city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_geo_city\",\"subType\":\"multi\"},{\"name\":\"source/ip_geo_country\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_geo_country.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_geo_country\",\"subType\":\"multi\"},{\"name\":\"source/ip_geo_country_iso\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_geo_country_iso.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_geo_country_iso\",\"subType\":\"multi\"},{\"name\":\"source/ip_geo_lat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_geo_lat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_geo_lat\",\"subType\":\"multi\"},{\"name\":\"source/ip_geo_lon\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_geo_lon.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_geo_lon\",\"subType\":\"multi\"},{\"name\":\"source/ip_ipreg_comment\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_ipreg_comment.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_ipreg_comment\",\"subType\":\"multi\"},{\"name\":\"source/ip_ipreg_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_ipreg_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_ipreg_domain\",\"subType\":\"multi\"},{\"name\":\"source/ip_ipreg_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source/ip_ipreg_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source/ip_ipreg_name\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tls.ja3.hash\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.ja3.hash.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.ja3.hash\",\"subType\":\"multi\"},{\"name\":\"tls.ja3.string\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.ja3.string.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.ja3.string\",\"subType\":\"multi\"},{\"name\":\"tls.sni\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.sni.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.sni\",\"subType\":\"multi\"},{\"name\":\"tls.sni_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.sni_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.sni_domain\",\"subType\":\"multi\"},{\"name\":\"tls.sni_domain_alexa\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.sni_domain_alexa.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.sni_domain_alexa\",\"subType\":\"multi\"},{\"name\":\"tls.sni_umbrella\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.sni_umbrella.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.sni_umbrella\",\"subType\":\"multi\"},{\"name\":\"tls.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tls.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tls.version\",\"subType\":\"multi\"}]","timeFieldName":"timestamp","title":"logs-suricata-tls-*"},"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T10:26:54.470Z","version":"Wzk4LDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata TLS - Histogram","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"Suricata TLS - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":{\"accessor\":0,\"format\":{\"id\":\"date\",\"params\":{\"pattern\":\"HH:mm\"}},\"params\":{\"date\":true,\"interval\":\"PT30M\",\"format\":\"HH:mm\",\"bounds\":{\"min\":\"2020-12-20T08:53:14.254Z\",\"max\":\"2020-12-21T08:53:14.254Z\"}},\"aggType\":\"date_histogram\"},\"y\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"080a28d0-436a-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:31:45.324Z","version":"WzExMCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata TLS - Top source IPs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata TLS - Top source IPs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP addr\"}}]}"},"id":"649dd8c0-436b-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T09:18:57.076Z","version":"WzgwLDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata TLS - Top destination IPs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata TLS - Top destination IPs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.ip.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Dest. IP addr\"}}]}"},"id":"e95d6ae0-436a-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T09:19:11.119Z","version":"WzgxLDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata TLS - Top SNI","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata TLS - Top SNI\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"tls.sni.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"SNI\"}}]}"},"id":"fbeb5370-436a-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:30:58.208Z","version":"WzEwNyw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata TLS - Top destination ports","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata TLS - Top destination ports\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"number\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Dest. port\"}}]}"},"id":"d5917220-436c-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:31:21.254Z","version":"WzEwOCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata TLS - TLS version","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Suricata TLS - TLS version\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"tls.version.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"f99b0560-436b-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:28:59.999Z","version":"WzEwMyw0XQ=="}
-{"attributes":{"columns":["in_iface","source.ip","destination.ip","destination.port","tls.version","tls.sni","tls.sni_domain_alexa","tls.sni_umbrella"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Suricata TLS logs","version":1},"id":"2fb21020-4377-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"74bb7bb0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T10:27:54.529Z","version":"Wzk5LDRd"}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":9,\"i\":\"bf273f44-b0c6-4a76-85ae-271bace06b5c\"},\"panelIndex\":\"bf273f44-b0c6-4a76-85ae-271bace06b5c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":9,\"w\":10,\"h\":15,\"i\":\"f3f513d2-c57c-402d-a7eb-9335533b9cee\"},\"panelIndex\":\"f3f513d2-c57c-402d-a7eb-9335533b9cee\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":10,\"y\":9,\"w\":11,\"h\":15,\"i\":\"280a69a1-470e-455f-a2af-e0f67a5b6899\"},\"panelIndex\":\"280a69a1-470e-455f-a2af-e0f67a5b6899\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":21,\"y\":9,\"w\":17,\"h\":15,\"i\":\"e2966d9f-3a5f-40c6-8046-921ca11dca36\"},\"panelIndex\":\"e2966d9f-3a5f-40c6-8046-921ca11dca36\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":38,\"y\":9,\"w\":10,\"h\":8,\"i\":\"8b66f551-7eea-46fd-a693-83291441986a\"},\"panelIndex\":\"8b66f551-7eea-46fd-a693-83291441986a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":38,\"y\":17,\"w\":10,\"h\":7,\"i\":\"91965e77-41d0-4046-b51b-acf16494b52c\"},\"panelIndex\":\"91965e77-41d0-4046-b51b-acf16494b52c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":24,\"w\":48,\"h\":22,\"i\":\"66a787fc-5c09-4a60-b878-5d453d6d5738\"},\"panelIndex\":\"66a787fc-5c09-4a60-b878-5d453d6d5738\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"}]","timeRestore":false,"title":"Suricata TLS","version":1},"id":"2d8baeb0-436c-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"080a28d0-436a-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"649dd8c0-436b-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"visualization"},{"id":"e95d6ae0-436a-11eb-b75a-bbebe0b50e97","name":"panel_2","type":"visualization"},{"id":"fbeb5370-436a-11eb-b75a-bbebe0b50e97","name":"panel_3","type":"visualization"},{"id":"d5917220-436c-11eb-b75a-bbebe0b50e97","name":"panel_4","type":"visualization"},{"id":"f99b0560-436b-11eb-b75a-bbebe0b50e97","name":"panel_5","type":"visualization"},{"id":"2fb21020-4377-11eb-b75a-bbebe0b50e97","name":"panel_6","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T10:30:30.675Z","version":"WzEwNSw0XQ=="}
-{"attributes":{"fieldFormatMap":"{\"ip_dst_addr_misp_url\":{\"id\":\"url\"}}","fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"alert.action\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert.action.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"alert.action\",\"subType\":\"multi\"},{\"name\":\"alert.category\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert.category.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"alert.category\",\"subType\":\"multi\"},{\"name\":\"alert.gid\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert.rev\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert.severity\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert.signature\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert.signature.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"alert.signature\",\"subType\":\"multi\"},{\"name\":\"alert.signature_id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"event_type\",\"subType\":\"multi\"},{\"name\":\"flow.bytes_toclient\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.bytes_toserver\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.pkts_toclient\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.pkts_toserver\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.start\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host\",\"subType\":\"multi\"},{\"name\":\"host_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host_domain\",\"subType\":\"multi\"},{\"name\":\"host_domain_freq1\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_domain_freq2\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_iface\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"in_iface.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"in_iface\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.accuracy\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_geo.geo.city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_geo.geo.city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_geo.geo.city\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.country.isoCode\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_geo.geo.country.isoCode.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_geo.geo.country.isoCode\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.country.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_geo.geo.country.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_geo.geo.country.name\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.latitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_geo.geo.longitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_misp\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_misp.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_misp\",\"subType\":\"multi\"},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.accuracy\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr_geo.geo.city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo.geo.city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo.geo.city\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.country.isoCode\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo.geo.country.isoCode.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo.geo.country.isoCode\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.country.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo.geo.country.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo.geo.country.name\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.latitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr_geo.geo.longitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"payload\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"payload.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"payload\",\"subType\":\"multi\"},{\"name\":\"payload_printable\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"payload_printable.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"payload_printable\",\"subType\":\"multi\"},{\"name\":\"proto\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"proto.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"proto\",\"subType\":\"multi\"},{\"name\":\"stream\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"vlan\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"logs-suricata-alert-*"},"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-20T21:20:28.734Z","version":"WzM1LDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata Alerts Histogram","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Suricata Alerts Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"71a37750-0b7c-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-20T14:01:02.393Z","version":"WzIzLDRd"}
-{"attributes":{"columns":["alert.signature_id","alert.signature","ip_src_addr","ip_dst_addr"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Suricata Alerts","version":1},"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"search":"7.4.0"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-20T14:01:02.393Z","version":"WzI0LDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Suricata alerts - top signatures","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata alerts - top signatures\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"alert.signature.keyword\",\"order\":\"desc\",\"size\":10,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Signature\"}}]}"},"id":"d7d96e70-0b7d-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-12-20T14:01:02.393Z","version":"WzI1LDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Suricata alerts - top ip_dst_addr ","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata alerts - top ip_dst_addr \",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"cardinality\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"alert.signature_id\",\"customLabel\":\"Unique\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr.keyword\",\"order\":\"desc\",\"size\":10,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Dst IP\"}}]}"},"id":"eb41e310-0b7e-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-12-20T14:01:02.393Z","version":"WzI2LDRd"}
-{"attributes":{"columns":["ip_dst_addr_misp","ip_dst_addr","alert.signature","ip_dst_addr_misp_url"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"ip_dst_addr_misp>0\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Suricata Alerts Misp","version":1},"id":"42ad6a30-15b0-11ea-841d-a1505e4ae442","migrationVersion":{"search":"7.4.0"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-20T14:01:02.393Z","version":"WzI3LDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Dst IP in misp","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Dst IP in misp\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Signature\"}}]}"},"id":"9676d8e0-15b0-11ea-841d-a1505e4ae442","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"42ad6a30-15b0-11ea-841d-a1505e4ae442","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-12-20T14:01:02.393Z","version":"WzI4LDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata alerts - the Hive","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Suricata alerts - the Hive\",\"type\":\"thehive_button\",\"params\":{\"url\":\"https://hive.soctools.geant.org/\",\"apikey\":\"ebMZixrFT+4qeWDf0iW3D5qFr/GbwA4j\",\"owner\":\"odfe\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"ip_src_addr.keyword\",\"order\":\"desc\",\"size\":20,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"48992900-62d3-11ea-aaa3-bb2f31340783","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-20T14:01:02.393Z","version":"WzI5LDRd"}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":7,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":16,\"w\":48,\"h\":14,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":7,\"w\":15,\"h\":9,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":15,\"y\":7,\"w\":9,\"h\":9,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":24,\"y\":7,\"w\":11,\"h\":9,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":39,\"y\":7,\"w\":9,\"h\":4,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"}]","timeRestore":false,"title":"Suricata Alerts","version":1},"id":"368ddb80-0b7f-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"71a37750-0b7c-11ea-bc07-2bc38b4c4b9b","name":"panel_0","type":"visualization"},{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"panel_1","type":"search"},{"id":"d7d96e70-0b7d-11ea-bc07-2bc38b4c4b9b","name":"panel_2","type":"visualization"},{"id":"eb41e310-0b7e-11ea-bc07-2bc38b4c4b9b","name":"panel_3","type":"visualization"},{"id":"9676d8e0-15b0-11ea-841d-a1505e4ae442","name":"panel_4","type":"visualization"},{"id":"48992900-62d3-11ea-aaa3-bb2f31340783","name":"panel_5","type":"visualization"}],"type":"dashboard","updated_at":"2020-12-21T21:58:35.823Z","version":"WzIxMSw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"class\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"class.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"class\",\"subType\":\"multi\"},{\"name\":\"level\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"level.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"level\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"source_file\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_file.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_file\",\"subType\":\"multi\"},{\"name\":\"source_host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_host\",\"subType\":\"multi\"},{\"name\":\"stackTrace\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"stackTrace.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"stackTrace\",\"subType\":\"multi\"},{\"name\":\"thread\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"thread.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"thread\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"logs-nifi-*"},"id":"635a5350-430a-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T10:42:10.466Z","version":"WzEyNiw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"NiFi Logs - Histogram","uiStateJSON":"{\"vis\":{\"colors\":{\"ERROR\":\"#BF1B00\",\"WARN\":\"#CCA300\",\"INFO\":\"#1F78C1\"}}}","version":1,"visState":"{\"title\":\"NiFi Logs - Histogram\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":{\"accessor\":0,\"format\":{\"id\":\"date\",\"params\":{\"pattern\":\"HH:mm\"}},\"params\":{\"date\":true,\"interval\":\"PT30M\",\"format\":\"HH:mm\",\"bounds\":{\"min\":\"2020-12-20T10:47:07.185Z\",\"max\":\"2020-12-21T10:47:07.185Z\"}},\"aggType\":\"date_histogram\"},\"y\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"bottom\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#34130C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"085d3790-437a-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"635a5350-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:49:20.127Z","version":"WzEyOSw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"NiFi logs - Source host","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"NiFi logs - Source host\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_host.keyword\",\"orderBy\":\"_key\",\"order\":\"asc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"3ad86f30-438b-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"635a5350-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:53:41.198Z","version":"WzIwNCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"NiFi logs - Level","uiStateJSON":"{\"vis\":{\"colors\":{\"ERROR\":\"#E24D42\",\"INFO\":\"#1F78C1\",\"WARN\":\"#CCA300\"}}}","version":1,"visState":"{\"title\":\"NiFi logs - Level\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"03184750-438b-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"635a5350-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T12:49:49.637Z","version":"WzEzMyw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"NiFI logs - source files","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"NiFI logs - source files\",\"type\":\"table\",\"params\":{\"perPage\":4,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":true,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_file.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source file\"}}]}"},"id":"e16c89f0-437a-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"635a5350-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:54:21.199Z","version":"WzEzMiw0XQ=="}
-{"attributes":{"columns":["source_host","level","source_file","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"NiFi logs","version":1},"id":"53a1d270-4379-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"635a5350-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T10:43:13.810Z","version":"WzEyNyw0XQ=="}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":11,\"i\":\"41579e7d-a833-4988-8e87-30e9934c9153\"},\"panelIndex\":\"41579e7d-a833-4988-8e87-30e9934c9153\",\"embeddableConfig\":{\"vis\":{\"colors\":{\"ERROR\":\"#E24D42\",\"WARN\":\"#CCA300\",\"INFO\":\"#1F78C1\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":11,\"w\":12,\"h\":10,\"i\":\"26a9e301-ca86-4313-8321-e5b8b67fa097\"},\"panelIndex\":\"26a9e301-ca86-4313-8321-e5b8b67fa097\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":12,\"y\":11,\"w\":9,\"h\":10,\"i\":\"0d515b73-44ae-48f0-9fbe-c330d044544a\"},\"panelIndex\":\"0d515b73-44ae-48f0-9fbe-c330d044544a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":21,\"y\":11,\"w\":13,\"h\":10,\"i\":\"9cf35fdc-5e6b-4a9b-a1fd-88c379d343da\"},\"panelIndex\":\"9cf35fdc-5e6b-4a9b-a1fd-88c379d343da\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":22,\"i\":\"e52833e0-30de-4451-80db-22c74ec92fcb\"},\"panelIndex\":\"e52833e0-30de-4451-80db-22c74ec92fcb\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]","timeRestore":false,"title":"NiFi logs","version":1},"id":"4b6ae5a0-437a-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"085d3790-437a-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"3ad86f30-438b-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"visualization"},{"id":"03184750-438b-11eb-b75a-bbebe0b50e97","name":"panel_2","type":"visualization"},{"id":"e16c89f0-437a-11eb-b75a-bbebe0b50e97","name":"panel_3","type":"visualization"},{"id":"53a1d270-4379-11eb-b75a-bbebe0b50e97","name":"panel_4","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T12:53:06.300Z","version":"WzEzNiw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"agent.ephemeral_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.ephemeral_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.ephemeral_id\",\"subType\":\"multi\"},{\"name\":\"agent.hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.hostname\",\"subType\":\"multi\"},{\"name\":\"agent.id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.id\",\"subType\":\"multi\"},{\"name\":\"agent.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.name\",\"subType\":\"multi\"},{\"name\":\"agent.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.type\",\"subType\":\"multi\"},{\"name\":\"agent.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.version\",\"subType\":\"multi\"},{\"name\":\"ecs.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ecs.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ecs.version\",\"subType\":\"multi\"},{\"name\":\"error.message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"error.message\",\"subType\":\"multi\"},{\"name\":\"error.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"error.name\",\"subType\":\"multi\"},{\"name\":\"error.stack\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.stack.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"error.stack\",\"subType\":\"multi\"},{\"name\":\"fields.log_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"fields.log_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"fields.log_type\",\"subType\":\"multi\"},{\"name\":\"host.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host.name\",\"subType\":\"multi\"},{\"name\":\"input.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"input.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"input.type\",\"subType\":\"multi\"},{\"name\":\"level\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"level.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"level\",\"subType\":\"multi\"},{\"name\":\"log.file.path\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"log.file.path.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"log.file.path\",\"subType\":\"multi\"},{\"name\":\"log.offset\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"metadata.beat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.beat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.beat\",\"subType\":\"multi\"},{\"name\":\"metadata.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.type\",\"subType\":\"multi\"},{\"name\":\"metadata.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.version\",\"subType\":\"multi\"},{\"name\":\"method\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"method.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"method\",\"subType\":\"multi\"},{\"name\":\"pid\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"prevMsg\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"prevMsg.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"prevMsg\",\"subType\":\"multi\"},{\"name\":\"prevState\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"prevState.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"prevState\",\"subType\":\"multi\"},{\"name\":\"req.headers.accept\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.accept-encoding\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.accept-encoding.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.accept-encoding\",\"subType\":\"multi\"},{\"name\":\"req.headers.accept-language\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.accept-language.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.accept-language\",\"subType\":\"multi\"},{\"name\":\"req.headers.accept.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.accept\",\"subType\":\"multi\"},{\"name\":\"req.headers.cache-control\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.cache-control.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.cache-control\",\"subType\":\"multi\"},{\"name\":\"req.headers.connection\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.connection.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.connection\",\"subType\":\"multi\"},{\"name\":\"req.headers.content-length\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.content-length.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.content-length\",\"subType\":\"multi\"},{\"name\":\"req.headers.content-type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.content-type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.content-type\",\"subType\":\"multi\"},{\"name\":\"req.headers.host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.host\",\"subType\":\"multi\"},{\"name\":\"req.headers.if-none-match\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.if-none-match.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.if-none-match\",\"subType\":\"multi\"},{\"name\":\"req.headers.kbn-version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.kbn-version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.kbn-version\",\"subType\":\"multi\"},{\"name\":\"req.headers.origin\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.origin.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.origin\",\"subType\":\"multi\"},{\"name\":\"req.headers.referer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.referer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.referer\",\"subType\":\"multi\"},{\"name\":\"req.headers.upgrade-insecure-requests\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.upgrade-insecure-requests.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.upgrade-insecure-requests\",\"subType\":\"multi\"},{\"name\":\"req.headers.user-agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.headers.user-agent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.headers.user-agent\",\"subType\":\"multi\"},{\"name\":\"req.method\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.method.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.method\",\"subType\":\"multi\"},{\"name\":\"req.referer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.referer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.referer\",\"subType\":\"multi\"},{\"name\":\"req.remoteAddress\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.remoteAddress.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.remoteAddress\",\"subType\":\"multi\"},{\"name\":\"req.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.url.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.url\",\"subType\":\"multi\"},{\"name\":\"req.userAgent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"req.userAgent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"req.userAgent\",\"subType\":\"multi\"},{\"name\":\"res.contentLength\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"res.responseTime\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"res.statusCode\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"state\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"state.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"state\",\"subType\":\"multi\"},{\"name\":\"statusCode\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"tags.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"tags\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"type\",\"subType\":\"multi\"}]","timeFieldName":"timestamp","title":"logs-kibana-*"},"id":"55426280-430a-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T16:15:49.869Z","version":"WzE0Nyw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Kibana logs - Histogram","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Kibana logs - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"84735610-43a8-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T16:21:02.064Z","version":"WzE0OCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Kibana logs - Top IPs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Kibana logs - Top IPs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"req.remoteAddress.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Remote address\"}}]}"},"id":"22b8f4d0-43cf-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T20:57:28.477Z","version":"WzE1Niw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Kibana logs - Response time","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"Kibana logs - Response time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{\"interval\":100},\"aggType\":\"histogram\"},\"y\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#34130C\",\"show\":false,\"style\":\"full\",\"value\":50,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"orderBucketsBySum\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"res.responseTime\",\"interval\":50,\"min_doc_count\":false,\"has_extended_bounds\":false,\"extended_bounds\":{\"max\":\"\",\"min\":\"\"},\"customLabel\":\"Response time [ms]\"}}]}"},"id":"90d03420-43ce-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T20:53:23.681Z","version":"WzE1NSw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Kibana logs - Status codes","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Kibana logs - Status codes\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"number\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"res.statusCode\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Status code\"}}]}"},"id":"8f67bff0-43cd-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T20:46:11.822Z","version":"WzE1NCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Kibana logs - Top URLs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Kibana logs - Top URLs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},{\"accessor\":3,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"req.url.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Requested URL\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"req.method.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Method\"}}]}"},"id":"d921f4a0-43a8-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T20:43:48.516Z","version":"WzE1Myw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Kibana logs - Top User-Agents","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Kibana logs - Top User-Agents\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"req.headers.user-agent.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User-Agent\"}}]}"},"id":"baef0cb0-43cc-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T20:40:15.354Z","version":"WzE1MSw0XQ=="}
-{"attributes":{"columns":["method","req.url","res.statusCode","req.remoteAddress","req.headers.user-agent"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Kibana logs","version":1},"id":"8f07c570-43a8-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"55426280-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T16:21:19.815Z","version":"WzE0OSw0XQ=="}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":8,\"i\":\"04285b5f-cd0f-4514-857c-0392c04dd759\"},\"panelIndex\":\"04285b5f-cd0f-4514-857c-0392c04dd759\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":8,\"w\":11,\"h\":12,\"i\":\"d6cba6b1-29d2-41bf-862b-85094bc155db\"},\"panelIndex\":\"d6cba6b1-29d2-41bf-862b-85094bc155db\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":11,\"y\":8,\"w\":23,\"h\":12,\"i\":\"2f0b8c11-89a2-4faa-bf5f-201803edae1d\"},\"panelIndex\":\"2f0b8c11-89a2-4faa-bf5f-201803edae1d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":34,\"y\":8,\"w\":14,\"h\":12,\"i\":\"99a7d5fc-d91f-4202-8c7e-48bfbf515084\"},\"panelIndex\":\"99a7d5fc-d91f-4202-8c7e-48bfbf515084\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":20,\"w\":28,\"h\":15,\"i\":\"7c98c844-9efd-4289-94b0-83101b21ee9b\"},\"panelIndex\":\"7c98c844-9efd-4289-94b0-83101b21ee9b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":28,\"y\":20,\"w\":20,\"h\":15,\"i\":\"08ce3890-961f-408f-9e2b-f9f028415e07\"},\"panelIndex\":\"08ce3890-961f-408f-9e2b-f9f028415e07\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":35,\"w\":48,\"h\":21,\"i\":\"0b4b31f9-53d5-4212-aedc-a261ee4be5e8\"},\"panelIndex\":\"0b4b31f9-53d5-4212-aedc-a261ee4be5e8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"}]","timeRestore":false,"title":"Kibana logs","version":1},"id":"6e0402e0-43cf-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"84735610-43a8-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"22b8f4d0-43cf-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"visualization"},{"id":"90d03420-43ce-11eb-b75a-bbebe0b50e97","name":"panel_2","type":"visualization"},{"id":"8f67bff0-43cd-11eb-b75a-bbebe0b50e97","name":"panel_3","type":"visualization"},{"id":"d921f4a0-43a8-11eb-b75a-bbebe0b50e97","name":"panel_4","type":"visualization"},{"id":"baef0cb0-43cc-11eb-b75a-bbebe0b50e97","name":"panel_5","type":"visualization"},{"id":"8f07c570-43a8-11eb-b75a-bbebe0b50e97","name":"panel_6","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T20:59:34.797Z","version":"WzE1Nyw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"agent.ephemeral_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.ephemeral_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.ephemeral_id\",\"subType\":\"multi\"},{\"name\":\"agent.hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.hostname\",\"subType\":\"multi\"},{\"name\":\"agent.id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.id\",\"subType\":\"multi\"},{\"name\":\"agent.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.name\",\"subType\":\"multi\"},{\"name\":\"agent.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.type\",\"subType\":\"multi\"},{\"name\":\"agent.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.version\",\"subType\":\"multi\"},{\"name\":\"ecs.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ecs.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ecs.version\",\"subType\":\"multi\"},{\"name\":\"event.severity\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"fields.log_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"fields.log_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"fields.log_type\",\"subType\":\"multi\"},{\"name\":\"host.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host.name\",\"subType\":\"multi\"},{\"name\":\"input.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"input.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"input.type\",\"subType\":\"multi\"},{\"name\":\"log.file.path\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"log.file.path.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"log.file.path\",\"subType\":\"multi\"},{\"name\":\"log.offset\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"log.source.address\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"log.source.address.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"log.source.address\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"metadata.beat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.beat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.beat\",\"subType\":\"multi\"},{\"name\":\"metadata.truncated\",\"type\":\"boolean\",\"esTypes\":[\"boolean\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"metadata.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.type\",\"subType\":\"multi\"},{\"name\":\"metadata.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.version\",\"subType\":\"multi\"},{\"name\":\"syslog.facility\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"syslog.facility_label\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"syslog.facility_label.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"syslog.facility_label\",\"subType\":\"multi\"},{\"name\":\"syslog.priority\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"syslog.severity_label\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"syslog.severity_label.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"syslog.severity_label\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"logs-filebeat-unknown-*"},"id":"b8cf4490-4309-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T21:44:27.290Z","version":"WzE5Myw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Other logs - Histogram","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Other logs - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":{\"accessor\":1,\"format\":{\"id\":\"date\",\"params\":{\"pattern\":\"YYYY-MM-DD HH:mm\"}},\"params\":{\"date\":true,\"interval\":\"PT3H\",\"format\":\"YYYY-MM-DD HH:mm\",\"bounds\":{\"min\":\"2020-12-14T21:46:19.383Z\",\"max\":\"2020-12-21T21:46:19.383Z\"}},\"aggType\":\"date_histogram\"},\"y\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"fields.log_type.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"f4775b50-43d5-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"b8cf4490-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:46:26.845Z","version":"WzE5Nyw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Other logs - Host name","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Other logs - Host name\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"host.name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"3579d010-43d6-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"b8cf4490-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:48:06.416Z","version":"WzE5OSw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Other logs - Log type","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Other logs - Log type\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"fields.log_type.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"1fe05530-43d6-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"b8cf4490-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:47:30.178Z","version":"WzE5OCw0XQ=="}
-{"attributes":{"columns":["fields.log_type","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Filebeat unknown logs","version":1},"id":"bfaaf800-43d5-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"b8cf4490-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T21:44:48.767Z","version":"WzE5NCw0XQ=="}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":9,\"i\":\"bf3a414b-96f0-4090-b163-43664f901493\"},\"panelIndex\":\"bf3a414b-96f0-4090-b163-43664f901493\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":9,\"w\":15,\"h\":9,\"i\":\"d6977944-5a19-48b6-8829-2e50838363e7\"},\"panelIndex\":\"d6977944-5a19-48b6-8829-2e50838363e7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":15,\"y\":9,\"w\":15,\"h\":9,\"i\":\"4cc8faa1-db1f-49e4-aaed-4e6010ff066b\"},\"panelIndex\":\"4cc8faa1-db1f-49e4-aaed-4e6010ff066b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":18,\"w\":48,\"h\":21,\"i\":\"09bccade-e7cc-455e-b5e7-af2403262ba6\"},\"panelIndex\":\"09bccade-e7cc-455e-b5e7-af2403262ba6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","timeRestore":false,"title":"Other logs","version":1},"id":"7104bb90-43d6-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"f4775b50-43d5-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"3579d010-43d6-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"visualization"},{"id":"1fe05530-43d6-11eb-b75a-bbebe0b50e97","name":"panel_2","type":"visualization"},{"id":"bfaaf800-43d5-11eb-b75a-bbebe0b50e97","name":"panel_3","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T21:49:46.312Z","version":"WzIwMCw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"agent.ephemeral_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.ephemeral_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.ephemeral_id\",\"subType\":\"multi\"},{\"name\":\"agent.hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.hostname\",\"subType\":\"multi\"},{\"name\":\"agent.id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.id\",\"subType\":\"multi\"},{\"name\":\"agent.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.name\",\"subType\":\"multi\"},{\"name\":\"agent.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.type\",\"subType\":\"multi\"},{\"name\":\"agent.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"agent.version\",\"subType\":\"multi\"},{\"name\":\"ecs.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ecs.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ecs.version\",\"subType\":\"multi\"},{\"name\":\"error.message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"error.message\",\"subType\":\"multi\"},{\"name\":\"error.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"error.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"error.type\",\"subType\":\"multi\"},{\"name\":\"fields.log_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"fields.log_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"fields.log_type\",\"subType\":\"multi\"},{\"name\":\"host.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host.name\",\"subType\":\"multi\"},{\"name\":\"hostName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"hostName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"hostName\",\"subType\":\"multi\"},{\"name\":\"input.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"input.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"input.type\",\"subType\":\"multi\"},{\"name\":\"level\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"level.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"level\",\"subType\":\"multi\"},{\"name\":\"log.file.path\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"log.file.path.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"log.file.path\",\"subType\":\"multi\"},{\"name\":\"log.offset\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"loggerClassName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"loggerClassName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"loggerClassName\",\"subType\":\"multi\"},{\"name\":\"loggerName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"loggerName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"loggerName\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"metadata.beat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.beat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.beat\",\"subType\":\"multi\"},{\"name\":\"metadata.type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.type\",\"subType\":\"multi\"},{\"name\":\"metadata.version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metadata.version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"metadata.version\",\"subType\":\"multi\"},{\"name\":\"ndc\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ndc.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ndc\",\"subType\":\"multi\"},{\"name\":\"processId\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"processName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"processName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"processName\",\"subType\":\"multi\"},{\"name\":\"sequence\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sourceClassName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sourceClassName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"sourceClassName\",\"subType\":\"multi\"},{\"name\":\"sourceFileName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sourceFileName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"sourceFileName\",\"subType\":\"multi\"},{\"name\":\"sourceLineNumber\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sourceMethodName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sourceMethodName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"sourceMethodName\",\"subType\":\"multi\"},{\"name\":\"sourceModuleName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sourceModuleName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"sourceModuleName\",\"subType\":\"multi\"},{\"name\":\"sourceModuleVersion\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sourceModuleVersion.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"sourceModuleVersion\",\"subType\":\"multi\"},{\"name\":\"threadId\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"threadName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"threadName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"threadName\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"logs-keycloak-*"},"id":"b9a340f0-430a-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T21:01:56.865Z","version":"WzE2MCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Keycloak logs - Histogram","uiStateJSON":"{\"vis\":{\"colors\":{\"INFO\":\"#1F78C1\"},\"legendOpen\":true}}","version":1,"visState":"{\"title\":\"Keycloak logs - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":{\"accessor\":1,\"format\":{\"id\":\"date\",\"params\":{\"pattern\":\"HH:mm\"}},\"params\":{\"date\":true,\"interval\":\"PT30M\",\"format\":\"HH:mm\",\"bounds\":{\"min\":\"2020-12-20T21:05:22.930Z\",\"max\":\"2020-12-21T21:05:22.930Z\"}},\"aggType\":\"date_histogram\"},\"y\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"level.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"4b5e4560-43d0-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"b9a340f0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:05:46.166Z","version":"WzE2Miw0XQ=="}
-{"attributes":{"columns":["level","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Keycloak logs","version":1},"id":"14c6e610-43d0-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"b9a340f0-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T21:04:14.577Z","version":"WzE2MSw0XQ=="}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":9,\"i\":\"f4adefb8-580f-47fa-a7b8-27be3af78eb1\"},\"panelIndex\":\"f4adefb8-580f-47fa-a7b8-27be3af78eb1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":9,\"w\":48,\"h\":20,\"i\":\"2f33edae-d9c7-4eaa-8e4d-494db3b1cf73\"},\"panelIndex\":\"2f33edae-d9c7-4eaa-8e4d-494db3b1cf73\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"}]","timeRestore":false,"title":"Keycloak logs","version":1},"id":"717b80f0-43d0-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"4b5e4560-43d0-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"14c6e610-43d0-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T21:06:50.111Z","version":"WzE2Myw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"actconn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"actconn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"actconn\",\"subType\":\"multi\"},{\"name\":\"backend_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"backend_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"backend_name\",\"subType\":\"multi\"},{\"name\":\"backend_queue\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"backend_queue.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"backend_queue\",\"subType\":\"multi\"},{\"name\":\"beconn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"beconn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"beconn\",\"subType\":\"multi\"},{\"name\":\"bytes_read\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"bytes_read.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"bytes_read\",\"subType\":\"multi\"},{\"name\":\"captured_request_cookie\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"captured_request_cookie.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"captured_request_cookie\",\"subType\":\"multi\"},{\"name\":\"captured_response_cookie\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"captured_response_cookie.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"captured_response_cookie\",\"subType\":\"multi\"},{\"name\":\"client.ip\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip\",\"subType\":\"multi\"},{\"name\":\"client.ip_geo_city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_geo_city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_geo_city\",\"subType\":\"multi\"},{\"name\":\"client.ip_geo_country\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_geo_country.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_geo_country\",\"subType\":\"multi\"},{\"name\":\"client.ip_geo_country_iso\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_geo_country_iso.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_geo_country_iso\",\"subType\":\"multi\"},{\"name\":\"client.ip_geo_lat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_geo_lat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_geo_lat\",\"subType\":\"multi\"},{\"name\":\"client.ip_geo_lon\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_geo_lon.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_geo_lon\",\"subType\":\"multi\"},{\"name\":\"client.ip_ipreg_comment\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_ipreg_comment.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_ipreg_comment\",\"subType\":\"multi\"},{\"name\":\"client.ip_ipreg_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_ipreg_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_ipreg_domain\",\"subType\":\"multi\"},{\"name\":\"client.ip_ipreg_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.ip_ipreg_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.ip_ipreg_name\",\"subType\":\"multi\"},{\"name\":\"client.port\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"client.port.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"client.port\",\"subType\":\"multi\"},{\"name\":\"feconn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"feconn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"feconn\",\"subType\":\"multi\"},{\"name\":\"frontend_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"frontend_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"frontend_name\",\"subType\":\"multi\"},{\"name\":\"http_host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_host\",\"subType\":\"multi\"},{\"name\":\"http_proto\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_proto.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_proto\",\"subType\":\"multi\"},{\"name\":\"http_request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_request.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_request\",\"subType\":\"multi\"},{\"name\":\"http_status_code\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_status_code.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_status_code\",\"subType\":\"multi\"},{\"name\":\"http_user\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_user.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_user\",\"subType\":\"multi\"},{\"name\":\"http_verb\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_verb.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_verb\",\"subType\":\"multi\"},{\"name\":\"http_version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_version\",\"subType\":\"multi\"},{\"name\":\"process.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"process.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"process.name\",\"subType\":\"multi\"},{\"name\":\"process.pid\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"process.pid.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"process.pid\",\"subType\":\"multi\"},{\"name\":\"retries\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"retries.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"retries\",\"subType\":\"multi\"},{\"name\":\"server.domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"server.domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"server.domain\",\"subType\":\"multi\"},{\"name\":\"source_host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_host\",\"subType\":\"multi\"},{\"name\":\"srv_queue\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"srv_queue.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"srv_queue\",\"subType\":\"multi\"},{\"name\":\"srvconn\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"srvconn.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"srvconn\",\"subType\":\"multi\"},{\"name\":\"termination_state\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"termination_state.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"termination_state\",\"subType\":\"multi\"},{\"name\":\"time_backend_connect\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"time_backend_connect.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"time_backend_connect\",\"subType\":\"multi\"},{\"name\":\"time_backend_response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"time_backend_response.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"time_backend_response\",\"subType\":\"multi\"},{\"name\":\"time_duration\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"time_duration.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"time_duration\",\"subType\":\"multi\"},{\"name\":\"time_queue\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"time_queue.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"time_queue\",\"subType\":\"multi\"},{\"name\":\"time_request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"time_request.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"time_request\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"logs-haproxy-*"},"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T21:27:29.655Z","version":"WzE3NSw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"haproxy logs - Histogram","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"haproxy logs - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":{\"accessor\":0,\"format\":{\"id\":\"date\",\"params\":{\"pattern\":\"HH:mm\"}},\"params\":{\"date\":true,\"interval\":\"PT30M\",\"format\":\"HH:mm\",\"bounds\":{\"min\":\"2020-12-20T21:34:47.670Z\",\"max\":\"2020-12-21T21:34:47.670Z\"}},\"aggType\":\"date_histogram\"},\"y\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"backend_name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"74693590-43d1-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:35:01.074Z","version":"WzE4MSw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"haproxy logs - Top clients","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"haproxy logs - Top clients\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"client.ip.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Client IP addr\"}}]}"},"id":"d2b6e700-43d1-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:16:42.736Z","version":"WzE2Niw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"haproxy logs - Top requests","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"haproxy logs - Top requests\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":3,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"backend_name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Backend name\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"http_request.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Request\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"http_status_code.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Status\"}}]}"},"id":"33158c30-43d4-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:38:09.088Z","version":"WzE4NCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"haproxy logs - Status codes","uiStateJSON":"{}","version":1,"visState":"{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"field\":\"http_status_code.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"size\":10},\"schema\":\"segment\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTooltip\":true,\"dimensions\":{\"buckets\":[{\"accessor\":0,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"Missing\",\"otherBucketLabel\":\"Other\"}},\"params\":{}}],\"metric\":{\"accessor\":1,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"params\":{}}},\"isDonut\":false,\"labels\":{\"last_level\":true,\"show\":false,\"truncate\":100,\"values\":true},\"legendPosition\":\"bottom\",\"type\":\"pie\"},\"title\":\"haproxy logs - Status codes\",\"type\":\"pie\"}"},"id":"96a9c140-43d3-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:29:37.571Z","version":"WzE3OCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"haproxy logs - Backends","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"haproxy logs - Backends\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"labels\":{\"last_level\":true,\"show\":false,\"truncate\":100,\"values\":true},\"legendPosition\":\"bottom\",\"type\":\"pie\",\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"backend_name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"2c952900-43d5-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:40:42.000Z","version":"WzE4OCw0XQ=="}
-{"attributes":{"columns":["client.ip","backend_name","http_verb","http_request","http_status_code","bytes_read","time_duration"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"haproxy logs","version":1},"id":"6c1be520-43d3-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"ece0e360-4309-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T21:28:09.585Z","version":"WzE3Niw0XQ=="}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":9,\"i\":\"ccce8e7d-f8cf-4074-929a-a5518428f22d\"},\"panelIndex\":\"ccce8e7d-f8cf-4074-929a-a5518428f22d\",\"embeddableConfig\":{\"legendOpen\":false,\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":9,\"w\":11,\"h\":16,\"i\":\"ba3bb2ae-8c2d-4d7c-b31b-68236235fa54\"},\"panelIndex\":\"ba3bb2ae-8c2d-4d7c-b31b-68236235fa54\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":11,\"y\":9,\"w\":28,\"h\":16,\"i\":\"07c2a778-1266-463e-9b3a-ea6f9d93e82b\"},\"panelIndex\":\"07c2a778-1266-463e-9b3a-ea6f9d93e82b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":39,\"y\":17,\"w\":9,\"h\":8,\"i\":\"d8e80787-da38-48a5-be9d-e73a94a1f0b3\"},\"panelIndex\":\"d8e80787-da38-48a5-be9d-e73a94a1f0b3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":39,\"y\":9,\"w\":9,\"h\":8,\"i\":\"efc8e873-81b6-46d4-91c8-003c1869de67\"},\"panelIndex\":\"efc8e873-81b6-46d4-91c8-003c1869de67\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":true},\"legendOpen\":false},\"panelRefName\":\"panel_4\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":25,\"w\":48,\"h\":22,\"i\":\"735a3ce2-b6f1-4d3b-af8f-05c4511e9b64\"},\"panelIndex\":\"735a3ce2-b6f1-4d3b-af8f-05c4511e9b64\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"}]","timeRestore":false,"title":"haproxy logs","version":1},"id":"7dfc0c10-43d4-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"74693590-43d1-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"d2b6e700-43d1-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"visualization"},{"id":"33158c30-43d4-11eb-b75a-bbebe0b50e97","name":"panel_2","type":"visualization"},{"id":"96a9c140-43d3-11eb-b75a-bbebe0b50e97","name":"panel_3","type":"visualization"},{"id":"2c952900-43d5-11eb-b75a-bbebe0b50e97","name":"panel_4","type":"visualization"},{"id":"6c1be520-43d3-11eb-b75a-bbebe0b50e97","name":"panel_5","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T21:42:37.188Z","version":"WzE4OSw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"audit_category\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_category.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_category\",\"subType\":\"multi\"},{\"name\":\"audit_cluster_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_cluster_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_cluster_name\",\"subType\":\"multi\"},{\"name\":\"audit_format_version\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"audit_node_host_address\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_node_host_address.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_node_host_address\",\"subType\":\"multi\"},{\"name\":\"audit_node_host_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_node_host_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_node_host_name\",\"subType\":\"multi\"},{\"name\":\"audit_node_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_node_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_node_id\",\"subType\":\"multi\"},{\"name\":\"audit_node_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_node_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_node_name\",\"subType\":\"multi\"},{\"name\":\"audit_request_exception_stacktrace\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_request_exception_stacktrace.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_request_exception_stacktrace\",\"subType\":\"multi\"},{\"name\":\"audit_request_layer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_request_layer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_request_layer\",\"subType\":\"multi\"},{\"name\":\"audit_request_origin\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"audit_request_origin.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"audit_request_origin\",\"subType\":\"multi\"}]","timeFieldName":"@timestamp","title":"security-auditlog-*"},"id":"cc44d890-430a-11eb-b75a-bbebe0b50e97","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-21T10:34:19.556Z","version":"WzExOCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Audit Log - Histogram","uiStateJSON":"{\"vis\":{\"legendOpen\":true}}","version":1,"visState":"{\"title\":\"Security Audit Log - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":{\"accessor\":0,\"format\":{\"id\":\"date\",\"params\":{\"pattern\":\"YYYY-MM-DD HH:mm\"}},\"params\":{\"date\":true,\"interval\":\"PT3H\",\"format\":\"YYYY-MM-DD HH:mm\",\"bounds\":{\"min\":\"2020-12-14T21:56:36.618Z\",\"max\":\"2020-12-21T21:56:36.618Z\"}},\"aggType\":\"date_histogram\"},\"y\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"audit_category.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"846b6f00-4375-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"cc44d890-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:56:51.161Z","version":"WzIwOCw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Audit Log - Node name","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Audit Log - Node name\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit_node_name.keyword\",\"orderBy\":\"1\",\"order\":\"asc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"b80e8d30-4378-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"cc44d890-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T21:55:05.416Z","version":"WzIwNyw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Audit Log - Category","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Audit Log - Category\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"audit_category.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"ca9c27a0-4378-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"cc44d890-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T10:39:23.930Z","version":"WzEyMSw0XQ=="}
-{"attributes":{"columns":["audit_node_host_name","audit_category","audit_request_layer","audit_request_origin","audit_request_exception_stacktrace"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Security Audit Logs","version":1},"id":"388503f0-4378-11eb-b75a-bbebe0b50e97","migrationVersion":{"search":"7.4.0"},"references":[{"id":"cc44d890-430a-11eb-b75a-bbebe0b50e97","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-12-21T10:35:18.830Z","version":"WzExOSw0XQ=="}
-{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":10,\"i\":\"963df476-bd2f-4c26-9652-3cdfa1eef34f\"},\"panelIndex\":\"963df476-bd2f-4c26-9652-3cdfa1eef34f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":10,\"w\":15,\"h\":9,\"i\":\"c87d79c0-84a1-46af-80a4-afc61cdae0a5\"},\"panelIndex\":\"c87d79c0-84a1-46af-80a4-afc61cdae0a5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":15,\"y\":10,\"w\":15,\"h\":9,\"i\":\"c32eb149-2f61-41b5-ae5a-a864fb3257cb\"},\"panelIndex\":\"c32eb149-2f61-41b5-ae5a-a864fb3257cb\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.4.2\",\"gridData\":{\"x\":0,\"y\":19,\"w\":48,\"h\":22,\"i\":\"cbd2ce7a-841d-4d11-b16e-79be174523e6\"},\"panelIndex\":\"cbd2ce7a-841d-4d11-b16e-79be174523e6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","timeRestore":false,"title":"Security Audit Log","version":1},"id":"e52ea260-4377-11eb-b75a-bbebe0b50e97","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"846b6f00-4375-11eb-b75a-bbebe0b50e97","name":"panel_0","type":"visualization"},{"id":"b80e8d30-4378-11eb-b75a-bbebe0b50e97","name":"panel_1","type":"visualization"},{"id":"ca9c27a0-4378-11eb-b75a-bbebe0b50e97","name":"panel_2","type":"visualization"},{"id":"388503f0-4378-11eb-b75a-bbebe0b50e97","name":"panel_3","type":"search"}],"type":"dashboard","updated_at":"2020-12-21T10:40:45.697Z","version":"WzEyMiw0XQ=="}
-{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"data_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"data_id\",\"subType\":\"multi\"},{\"name\":\"data_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"data_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"data_type\",\"subType\":\"multi\"},{\"name\":\"http_version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_version\",\"subType\":\"multi\"},{\"name\":\"ident\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ident.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ident\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_city\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_country\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_country.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_country\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_country_iso\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_country_iso.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_country_iso\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_lat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_lat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_lat\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_lon\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_lon.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_lon\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_ipreg_comment\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_ipreg_comment.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_ipreg_comment\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_ipreg_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_ipreg_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_ipreg_domain\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_ipreg_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_ipreg_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_ipreg_name\",\"subType\":\"multi\"},{\"name\":\"method\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"method.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"method\",\"subType\":\"multi\"},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"referer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"referer\",\"subType\":\"multi\"},{\"name\":\"request_page\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request_page.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"request_page\",\"subType\":\"multi\"},{\"name\":\"response_size\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response_size.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"response_size\",\"subType\":\"multi\"},{\"name\":\"server_response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"server_response.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"server_response\",\"subType\":\"multi\"},{\"name\":\"source_file\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_file.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_file\",\"subType\":\"multi\"},{\"name\":\"source_host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_host\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"user_agent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"user_agent\",\"subType\":\"multi\"}]","timeFieldName":"timestamp","title":"logs-nginx-*"},"id":"4ca554f0-893f-11ea-977f-4711a028b7c3","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-12-20T14:01:02.393Z","version":"WzMxLDRd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Nginx Logs - Histogram","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Nginx Logs - Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#34130C\"},\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"5bafcf20-43a7-11eb-b75a-bbebe0b50e97","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"4ca554f0-893f-11ea-977f-4711a028b7c3","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-12-21T16:12:44.178Z","version":"WzEzOSw0XQ=="}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Suricata alerts - top ip_src_addr ","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata alerts - top ip_src_addr \",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"cardinality\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"alert.signature_id\",\"customLabel\":\"Unique\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_src_addr.keyword\",\"order\":\"desc\",\"size\":10,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Src IP\"}}]}"},"id":"d8322050-0b7e-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-12-20T14:01:02.393Z","version":"WzMyLDRd"}
\ No newline at end of file
+{"attributes":{"fieldFormatMap":"{\"ip_dst_addr_misp_url\":{\"id\":\"url\"}}","fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"alert.action\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert.action.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"alert.action\",\"subType\":\"multi\"},{\"name\":\"alert.category\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert.category.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"alert.category\",\"subType\":\"multi\"},{\"name\":\"alert.gid\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert.rev\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert.severity\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"alert.signature\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"alert.signature.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"alert.signature\",\"subType\":\"multi\"},{\"name\":\"alert.signature_id\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"event_type\",\"subType\":\"multi\"},{\"name\":\"flow.bytes_toclient\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.bytes_toserver\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.pkts_toclient\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.pkts_toserver\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"flow.start\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host\",\"subType\":\"multi\"},{\"name\":\"host_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"host_domain\",\"subType\":\"multi\"},{\"name\":\"host_domain_freq1\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host_domain_freq2\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"in_iface\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"in_iface.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"in_iface\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.accuracy\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_geo.geo.city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_geo.geo.city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_geo.geo.city\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.country.isoCode\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_geo.geo.country.isoCode.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_geo.geo.country.isoCode\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.country.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_geo.geo.country.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_geo.geo.country.name\",\"subType\":\"multi\"},{\"name\":\"ip_dst_addr_geo.geo.latitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_geo.geo.longitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_misp\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_dst_addr_misp.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_dst_addr_misp\",\"subType\":\"multi\"},{\"name\":\"ip_dst_port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.accuracy\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr_geo.geo.city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo.geo.city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo.geo.city\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.country.isoCode\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo.geo.country.isoCode.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo.geo.country.isoCode\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.country.name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo.geo.country.name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo.geo.country.name\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo.geo.latitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_addr_geo.geo.longitude\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_src_port\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"payload\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"payload.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"payload\",\"subType\":\"multi\"},{\"name\":\"payload_printable\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"payload_printable.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"payload_printable\",\"subType\":\"multi\"},{\"name\":\"proto\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"proto.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"proto\",\"subType\":\"multi\"},{\"name\":\"stream\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"vlan\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ip_dst_addr_misp_url\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"if (!doc['ip_dst_addr_misp.keyword'].empty) { \\n return 'https://misp.soctools.geant.org/events/view/'+doc['ip_dst_addr_misp.keyword'].value;\\n}\\nreturn null;\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]","timeFieldName":"timestamp","title":"logs-suricata-alert-*"},"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-07-08T12:42:14.207Z","version":"WzMsMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata Alerts Histogram","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Suricata Alerts Histogram\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"id":"71a37750-0b7c-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-07-08T12:42:14.207Z","version":"WzQsMV0="}
+{"attributes":{"columns":["alert.signature_id","alert.signature","ip_src_addr","ip_dst_addr"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Suricata Alerts","version":1},"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"search":"7.4.0"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-07-08T12:42:14.207Z","version":"WzUsMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Suricata alerts - top signatures","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata alerts - top signatures\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"alert.signature.keyword\",\"order\":\"desc\",\"size\":10,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Signature\"}}]}"},"id":"d7d96e70-0b7d-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-07-08T12:42:14.207Z","version":"WzYsMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Suricata alerts - top ip_dst_addr ","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata alerts - top ip_dst_addr \",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"cardinality\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"alert.signature_id\",\"customLabel\":\"Unique\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr.keyword\",\"order\":\"desc\",\"size\":10,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Dst IP\"}}]}"},"id":"eb41e310-0b7e-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-07-08T12:42:14.207Z","version":"WzcsMV0="}
+{"attributes":{"columns":["ip_dst_addr_misp","ip_dst_addr","alert.signature","ip_dst_addr_misp_url"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"ip_dst_addr_misp>0\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["timestamp","desc"]],"title":"Suricata Alerts Misp","version":1},"id":"42ad6a30-15b0-11ea-841d-a1505e4ae442","migrationVersion":{"search":"7.4.0"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-07-08T12:42:14.207Z","version":"WzgsMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Dst IP in misp","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Dst IP in misp\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_dst_addr.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Signature\"}}]}"},"id":"9676d8e0-15b0-11ea-841d-a1505e4ae442","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"42ad6a30-15b0-11ea-841d-a1505e4ae442","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-07-08T12:42:14.207Z","version":"WzksMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Suricata alerts - the Hive","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Suricata alerts - the Hive\",\"type\":\"thehive_button\",\"params\":{\"url\":\"https://hive.soctools.geant.org/\",\"apikey\":\"ebMZixrFT+4qeWDf0iW3D5qFr/GbwA4j\",\"owner\":\"odfe\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"ip_src_addr.keyword\",\"order\":\"desc\",\"size\":20,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"48992900-62d3-11ea-aaa3-bb2f31340783","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"e81e23f0-0b75-11ea-bc07-2bc38b4c4b9b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-07-08T12:42:14.207Z","version":"WzEwLDFd"}
+{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":7,\"i\":\"1\"},\"panelIndex\":\"1\",\"version\":\"7.3.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":15,\"w\":48,\"h\":14,\"i\":\"2\"},\"panelIndex\":\"2\",\"version\":\"7.3.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":7,\"w\":10,\"h\":8,\"i\":\"3\"},\"panelIndex\":\"3\",\"version\":\"7.3.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":10,\"y\":7,\"w\":9,\"h\":8,\"i\":\"5\"},\"panelIndex\":\"5\",\"version\":\"7.3.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":19,\"y\":7,\"w\":11,\"h\":8,\"i\":\"7\"},\"panelIndex\":\"7\",\"version\":\"7.3.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":39,\"y\":7,\"w\":9,\"h\":3,\"i\":\"8\"},\"panelIndex\":\"8\",\"version\":\"7.3.0\",\"panelRefName\":\"panel_5\"}]","timeRestore":false,"title":"Suricata Alerts","version":1},"id":"368ddb80-0b7f-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"71a37750-0b7c-11ea-bc07-2bc38b4c4b9b","name":"panel_0","type":"visualization"},{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"panel_1","type":"search"},{"id":"d7d96e70-0b7d-11ea-bc07-2bc38b4c4b9b","name":"panel_2","type":"visualization"},{"id":"eb41e310-0b7e-11ea-bc07-2bc38b4c4b9b","name":"panel_3","type":"visualization"},{"id":"9676d8e0-15b0-11ea-841d-a1505e4ae442","name":"panel_4","type":"visualization"},{"id":"48992900-62d3-11ea-aaa3-bb2f31340783","name":"panel_5","type":"visualization"}],"type":"dashboard","updated_at":"2020-07-08T12:42:14.207Z","version":"WzExLDFd"}
+{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"data_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"data_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"data_id\",\"subType\":\"multi\"},{\"name\":\"data_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"data_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"data_type\",\"subType\":\"multi\"},{\"name\":\"http_version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"http_version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"http_version\",\"subType\":\"multi\"},{\"name\":\"ident\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ident.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ident\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_city\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_country\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_country.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_country\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_country_iso\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_country_iso.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_country_iso\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_lat\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_lat.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_lat\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_geo_lon\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_geo_lon.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_geo_lon\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_ipreg_comment\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_ipreg_comment.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_ipreg_comment\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_ipreg_domain\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_ipreg_domain.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_ipreg_domain\",\"subType\":\"multi\"},{\"name\":\"ip_src_addr_ipreg_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ip_src_addr_ipreg_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ip_src_addr_ipreg_name\",\"subType\":\"multi\"},{\"name\":\"method\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"method.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"method\",\"subType\":\"multi\"},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"referer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"referer\",\"subType\":\"multi\"},{\"name\":\"request_page\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request_page.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"request_page\",\"subType\":\"multi\"},{\"name\":\"response_size\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response_size.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"response_size\",\"subType\":\"multi\"},{\"name\":\"server_response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"server_response.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"server_response\",\"subType\":\"multi\"},{\"name\":\"source_file\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_file.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_file\",\"subType\":\"multi\"},{\"name\":\"source_host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_host.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"source_host\",\"subType\":\"multi\"},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"user_agent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"user_agent\",\"subType\":\"multi\"}]","timeFieldName":"timestamp","title":"logs-nginx-*"},"id":"4ca554f0-893f-11ea-977f-4711a028b7c3","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2020-07-08T12:42:14.207Z","version":"WzEyLDFd"}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Suricata alerts - top ip_src_addr ","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Suricata alerts - top ip_src_addr \",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"cardinality\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"alert.signature_id\",\"customLabel\":\"Unique\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ip_src_addr.keyword\",\"order\":\"desc\",\"size\":10,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Src IP\"}}]}"},"id":"d8322050-0b7e-11ea-bc07-2bc38b4c4b9b","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"35141420-0b7c-11ea-bc07-2bc38b4c4b9b","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-07-08T12:42:14.207Z","version":"WzEzLDFd"}
\ No newline at end of file
diff --git a/roles/odfekibana/tasks/main.yml b/roles/odfekibana/tasks/main.yml
index 1662efd600ad1a43e47bc3c40acb09fe69599dda..c0854140c7d172dc700f010fb756728b5c296f55 100644
--- a/roles/odfekibana/tasks/main.yml
+++ b/roles/odfekibana/tasks/main.yml
@@ -1,17 +1,184 @@
---
-- include: start.yml
+#- name: Create config directory
+# file:
+# name: config
+# state: directory
+# mode: 0700
+# tags:
+# - start
+
+- name: Copy cacert to ca-trust dir
+ remote_user: root
+ copy:
+ src: "files/{{ca_cn}}.crt"
+ dest: /etc/pki/ca-trust/source/anchors/ca.crt
+ tags:
+ - start
+
+- name: Install cacert to root truststore
+ remote_user: root
+ command: "update-ca-trust"
+ tags:
+ - start
+
+- name: Copy certificates in odfe kibana conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "config/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.p12"
+ - "{{ inventory_hostname }}.crt"
+ - "{{ inventory_hostname }}.key"
+ - cacerts.jks
+ - "{{ca_cn}}.crt"
+ - "{{soctools_users[0].CN}}.p12"
tags:
- - start
-- include: stop.yml
+ - start
+
+- name: Get openid authkey
+ set_fact:
+ kibanasecret: "{{lookup('file', 'files/kibanasecret',convert_data=False) | from_json }}"
+ tags:
+ - start
+
+#- name: Configure sysconfig
+# template:
+# src: sysconfig_elasticsearch.j2
+# dest: sysconfig_elasticsearch
+# tags:
+# - start
+#
+#- name: Copy sysconfig to /etc
+# command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
+# tags:
+# - start
+
+# lineinfile:
+# path: /etc/sysconfig/elasticsearch
+# regexp: '^ES_PATH_CONF='
+# line: ES_PATH_CONF=/usr/share/elasticsearch/config
+# tags:
+# - start
+- name: Configure odfe kibana properties
+ template:
+ src: "{{item}}.j2"
+ dest: "config/{{item}}"
+ with_items:
+ - kibana.yml
tags:
- - stop
- - stop-odfekibana
-- include: update-config.yml
+ - start
+
+- name: Configure odfe kibana start script
+ template:
+ src: "{{item}}.j2"
+ dest: "{{item}}"
+ mode: 0750
+ with_items:
+ - startkibana.sh
+ tags:
+ - start
+
+#- name: Exit here to test ODFE
+# meta: end_play
+# tags:
+# - start
+
+
+- name: Generate configuration for thehive_button plugin
+ template:
+ src: files/env.js.j2
+ dest: "/usr/share/kibana/plugins/thehive_button/public/env.js"
+ owner: kibana
+ group: kibana
+ tags:
+ - start
+
+
+- name: Start OpenDistro Kibana for Elasticsearch
+ command: /usr/share/kibana/startkibana.sh
+ #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml &
+ #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &"
tags:
- - update-config
- - update-odfekibana-config
-- include: restart.yml
+ - start
+
+- name: Wait for Kibana
+ wait_for:
+ host: "{{groups['odfekibanacontainers'][0]}}"
+ port: 5601
+ state: started
+ delay: 5
+ tags:
+ - start
+
+- name: Check Kibana health
+ shell: 'curl -k -b /tmp/cookie.txt -c /tmp/cookie.txt -X "GET" "https://{{dslproxy}}:5601/api/status" \
+ | egrep status....overall....state...green'
+ register: result
+ until: result.rc == 0
+ retries: 90
+ delay: 2
+ ignore_errors: yes
+ tags:
+ - start
+
+- name: Copy tenant.json to container
+ remote_user: kibana
+ copy:
+ src: "files/tenant.json"
+ dest: /tmp/tenant.json
+ tags:
+ - start
+
+- name: change tenant to global
+ shell: 'curl -X "POST" "https://{{dslproxy}}:5601/api/v1/multitenancy/tenant" \
+ -b /tmp/cookie.txt -c /tmp/cookie.txt \
+ -k --user admin:{{ odfees_adminpass }} \
+ -H "kbn-xsrf: reporting" -H "Content-Type: application/json" \
+ -d @/tmp/tenant.json'
+ tags:
+ - start
+
+- name: Copy kibana_graphs.ndjson to container
+ remote_user: kibana
+ copy:
+ src: "files/kibana_graphs.ndjson"
+ dest: /tmp/kibana_graphs.ndjson
+ tags:
+ - start
+
+- name: Import graphs to kibana
+ shell: 'curl -X "POST" "https://{{dslproxy}}:5601/api/saved_objects/_import?overwrite=true" \
+ -b /tmp/cookie.txt -c /tmp/cookie.txt \
+ -k --user admin:{{ odfees_adminpass }} \
+ -H "kbn-xsrf: reporting" -H "Content-Type: multipart/form-data" \
+ -F "file=@/tmp/kibana_graphs.ndjson"'
+ tags:
+ - start
+
+#- name: cleanup temporary files for kibana_graph import
+# shell: '/bin/rm -rf /tmp/cookie.txt /tmp/kibana_graphs.ndjson /tmp/tenant.json'
+# ignore_errors: true
+# tags:
+# - start
+
+#- name: check reachable hosts
+# gather_facts: no
+# tasks:
+# - command: ping -c1 {{ inventory_hostname }}
+# delegate_to: localhost
+# register: ping_result
+# ignore_errors: yes
+# - group_by: key=reachable
+# when: ping_result|success
+
+#- name: Stop OpenDistro Kibana for Elasticsearch
+# command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+# tags:
+# - stop
+
+- name: Stop OpenDistro Kibana for Elasticsearch
+ command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
tags:
- - restart
- - restart-odfekibana
+ - stop
diff --git a/roles/odfekibana/templates/kibana.yml.j2 b/roles/odfekibana/templates/kibana.yml.j2
index e676b3735c0f2f9ee59a2c7b3e19c020e04faa1b..0d670073c5d8f8515075abcea316f02d4863df25 100644
--- a/roles/odfekibana/templates/kibana.yml.j2
+++ b/roles/odfekibana/templates/kibana.yml.j2
@@ -29,8 +29,8 @@ elasticsearch.username: kibanaserver
elasticsearch.password: kibanaserver
elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"]
-opendistro_security.multitenancy.enabled: false
-#opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
+opendistro_security.multitenancy.enabled: true
+opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
opendistro_security.readonly_mode.roles: ["kibana_read_only"]
#new in 7.6
@@ -39,20 +39,20 @@ opendistro_security.readonly_mode.roles: ["kibana_read_only"]
#telemetry.enabled: false
opendistro_security.auth.type: "openid"
-opendistro_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
-opendistro_security.openid.client_id: "soctools-kibana"
+opendistro_security.openid.connect_url: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
+opendistro_security.openid.client_id: "dsoclab-kibana"
opendistro_security.openid.client_secret: "{{kibanasecret.value}}"
-opendistro_security.openid.root_ca: "/usr/share/kibana/config/ca.crt"
-opendistro_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601"
+opendistro_security.openid.root_ca: "/usr/share/kibana/config/{{ca_cn}}.crt"
+opendistro_security.openid.base_redirect_url: "https://{{dslproxy}}:5601"
opendistro_security.cookie.secure: true
-opendistro_security.cookie.password: {{lookup("password", "{{playbook_dir}}/secrets/passwords/kibana_cookiepassword length=32")}}
+opendistro_security.cookie.password: "{{odfekibana_cookie}}"
server.ssl.enabled: true
server.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
server.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
#server.ssl.keystore.path: /usr/share/kibana/config/{{inventory_hostname}}.p12
-#server.ssl.keystore.password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
+#server.ssl.keystore.password: {{kspass}}
#server.ssl.certificateAuthorities:
#server.ssl.truststore.path: jks (p12?)
#server.ssl.truststore.password:
diff --git a/roles/soctools-server/tasks/main.yml b/roles/soctools-server/tasks/main.yml
index 98902dce701cf0b9f7b6d57c6b3fd5df601a85e9..55adbbdb04ed879233a41f8b11d6219358147a17 100644
--- a/roles/soctools-server/tasks/main.yml
+++ b/roles/soctools-server/tasks/main.yml
@@ -16,7 +16,7 @@
- name: Install required packages
yum:
- name: ["docker-ce","docker-ce-cli","containerd.io","python-pip","unzip"]
+ name: ["docker-ce","docker-ce-cli","containerd.io","python-pip"]
state: latest
validate_certs: no
when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'
diff --git a/roles/thehive/tasks/main.yml b/roles/thehive/tasks/main.yml
index a0f7edf3191254869b03c63056db742bacb7b71e..0e560e7122b33b36a8ab31e0b0c6f6c46570153f 100644
--- a/roles/thehive/tasks/main.yml
+++ b/roles/thehive/tasks/main.yml
@@ -1,18 +1,87 @@
---
-- include: start.yml
+- name: Copy cacert to ca-trust dir
+ remote_user: root
+ copy:
+ src: "files/{{ca_cn}}.crt"
+ dest: /etc/pki/ca-trust/source/anchors/ca.crt
tags:
- - start
-- include: stop.yml
+ - start
+
+- name: Install cacert to root truststore
+ remote_user: root
+ command: "update-ca-trust"
+ tags:
+ - start
+
+- name: Copy certificates in thehive conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "/etc/thehive/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.crt"
+ - "{{ inventory_hostname }}.key"
+ - cacerts.jks
+ - "{{ca_cn}}.crt"
+ tags:
+ - start
+
+- name: Get openid authkey
+ set_fact:
+ thehivesecret: "{{lookup('file', 'files/thehivesecret',convert_data=False) | from_json }}"
tags:
- - stop
- - stop-thehive
-- include: update-config.yml
+ - start
+
+
+- name: Configure TheHive
+ template:
+ src: application.conf.j2
+ dest: /etc/thehive/application.conf
+ tags:
+ - start
+
+- name: Configure TheHive logging
+ copy:
+ src: logback.xml
+ dest: /etc/thehive/logback.xml
tags:
- - update-config
- - update-thehive-config
-- include: restart.yml
+ - start
+
+
+- name: Start TheHive
+ command: >
+ daemonize
+ -c /opt/thehive
+ -p /tmp/thehive.pid
+ -o /tmp/thehive-stdout.log
+ /opt/thehive/bin/thehive
+ -Dconfig.file=/etc/thehive/application.conf
+ -Dlogger.file=/etc/thehive/logback.xml
+ -J-Xms1g
+ -J-Xmx1g
+ -Dpidfile.path=/dev/null
+ tags:
+ - start
+
+- name: Wait for TheHive
+ wait_for:
+ host: "{{groups['thehive'][0]}}"
+ port: 9000
+ state: started
+ delay: 5
+ tags:
+ - start
+
+- name: Create TheHive users
+ include: createusers.yml
+ tags:
+ - createusers
+ - start
+
+- name: Stop TheHive
+ command: "pkill -SIGTERM -F /tmp/thehive.pid"
tags:
- - restart
- - restart-thehive
+ - stop
+ - stopthehive
diff --git a/roles/thehive/templates/application.conf.j2 b/roles/thehive/templates/application.conf.j2
index d25e059818a773a6a4a48fb34196cc641f79e718..a92e4f71ca871305611695aa04367054738881d0 100644
--- a/roles/thehive/templates/application.conf.j2
+++ b/roles/thehive/templates/application.conf.j2
@@ -5,7 +5,7 @@
## Include Play secret key
# More information on secret key at https://www.playframework.com/documentation/2.8.x/ApplicationSecret
#include "/etc/thehive/secret.conf"
-play.http.secret.key="{{lookup('password', '{{playbook_dir}}/secrets/passwords/thehive_secret_key')}}"
+play.http.secret.key="{{thehive_secret_key}}"
## Database configuration
db.janusgraph {
@@ -13,7 +13,7 @@ db.janusgraph {
## Cassandra configuration
# More information at https://docs.janusgraph.org/basics/configuration-reference/#storagecql
backend: cql
- hostname: ["{{groups['cassandra'][0]}}.{{soctools_netname}}"]
+ hostname: ["{{groups['cassandra'][0]}}.{{soctools_netname}}:9042"]
# Cassandra authentication (if configured)
// username: "thehive"
// password: "password"
@@ -47,17 +47,61 @@ storage {
## Authentication configuration
# More information at https://github.com/TheHive-Project/TheHiveDocs/TheHive4/Administration/Authentication.md
-//auth {
-// providers: [
+auth {
+ providers: [
// {name: session} # required !
// {name: basic, realm: thehive}
// {name: local}
// {name: key}
-// ]
+ {name: session} # required !
+ {name: basic, realm: thehive}
+ {name: local}
+ {name: key}
+ {
+ name: oauth2
+ clientId: "dsoclab-thehive"
+ clientSecret: {{thehivesecret.value}}
+ redirectUri: "https://{{dslproxy}}:9000/api/ssoLogin"
+ responseType: "code"
+ grantType: "authorization_code"
+ authorizationUrl: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/auth"
+ authorizationHeader: "Bearer"
+ tokenUrl: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/token"
+ userUrl: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/userinfo"
+// scope: ["openid", "email"]
+ scope: ["openid"]
+ userIdField: "email"
+// userIdField: "name"
+ }
+ ]
+ sso {
+ autocreate: true
+ autoupdate: true
+ autologin: true
+ mapper: "simple"
+// attributes {
+// login: "login"
+// name: "name"
+// roles: "role"
+// }
+ defaultRoles: ["read", "write", "admin"]
+ defaultOrganization: "uninett.no"
+// defaultOrganization: "demo"
+ }
+ ws.ssl.trustManager {
+ stores = [
+ {
+ type: "JKS" // JKS or PEM
+ path: "cacerts.jks"
+ password: "{{tspass}}"
+ }
+ ]
+ }
# The format of logins must be valid email address format. If the provided login doesn't contain `@` the following
# domain is automatically appended
-// defaultUserDomain: "thehive.local"
-//}
+ defaultUserDomain: "uninett.no"
+# defaultUserDomain: "thehive.local"
+}
## CORTEX configuration
# More information at https://github.com/TheHive-Project/TheHiveDocs/TheHive4/Administration/Connectors.md
diff --git a/soctools.yml b/soctools.yml
index fb4a2196557009b772ebf937cc80fe2658d65001..c25b56f32c95d5973e64ce70d6a1d7b4c3eae539 100644
--- a/soctools.yml
+++ b/soctools.yml
@@ -2,16 +2,9 @@
- name: Start soctools cluster
import_playbook: startsoctools.yml
- when: "'start' in ansible_run_tags or 'config' in ansible_run_tags"
+ when: "'start' in ansible_run_tags"
- name: Stop soctools cluster
import_playbook: stopsoctools.yml
- when: "'stop' in ansible_run_tags or 'stop-thehive' in ansible_run_tags or 'stop-keycloak' in ansible_run_tags or 'stop-cortex' in ansible_run_tags or 'stop-haproxy' in ansible_run_tags or 'stop-cassandra' in ansible_run_tags or 'stop-filebeat' in ansible_run_tags or 'stop-misp' in ansible_run_tags or 'stop-mysql' in ansible_run_tags or 'stop-nifi' in ansible_run_tags or 'stop-odfees' in ansible_run_tags or 'stop-odfekibana' in ansible_run_tags"
+ when: "'stop' in ansible_run_tags"
-- name: Update soctools cluster configs
- import_playbook: update-config-soctools.yml
- when: "'update-config' in ansible_run_tags or 'update-keycloak-config' in ansible_run_tags or 'update-thehive-config' in ansible_run_tags or 'update-cortex-config' in ansible_run_tags or 'update-haproxy-config' in ansible_run_tags or 'update-cassandra-config' in ansible_run_tags or 'update-filebeat-config' in ansible_run_tags or 'update-misp-config' in ansible_run_tags or 'update-mysql-config' in ansible_run_tags or 'update-nifi-config' in ansible_run_tags or 'update-odfees-config' in ansible_run_tags or 'update-odfekibana-config' in ansible_run_tags"
-
-- name: restart soctools cluster servics
- import_playbook: restart-soctools.yml
- when: "'restart' in ansible_run_tags or 'restart-thehive' in ansible_run_tags or 'restart-keycloak' in ansible_run_tags or 'restart-cortex' in ansible_run_tags or 'restart-haproxy' in ansible_run_tags or 'restart-cassandra' in ansible_run_tags or 'restart-filebeat' in ansible_run_tags or 'restart-misp' in ansible_run_tags or 'restart-mysql' in ansible_run_tags or 'restart-nifi' in ansible_run_tags or 'restart-odfees' in ansible_run_tags or 'restart-odfekibana' in ansible_run_tags"
diff --git a/soctools_server.yml b/soctools_server.yml
index 382d998567a630c58950d28fd43802eaf31089d9..23716c5a7e7d32fdc7bedd0faf775afb5c55bfbc 100644
--- a/soctools_server.yml
+++ b/soctools_server.yml
@@ -1,5 +1,5 @@
---
-- hosts: soctoolsmain
+- hosts: dsldev
become: true
roles:
- soctools-server
diff --git a/startsoctools.yml b/startsoctools.yml
index c9c4c6bb95760d4da7a0a46706507bd4d5867174..f1154c340f91f5604fd36c1abd38f9cc718c58f0 100644
--- a/startsoctools.yml
+++ b/startsoctools.yml
@@ -1,7 +1,7 @@
---
- name: Start docker containers
- hosts: soctoolsmain
+ hosts: dsldev
roles:
- docker
@@ -25,11 +25,6 @@
roles:
- keycloak
-- name: Reconfigure and start MISP
- hosts: mispcontainers
- roles:
- - misp
-
- name: Reconfigure and start NiFi
hosts: nificontainers
roles:
@@ -45,10 +40,10 @@
roles:
- odfekibana
-- name: Install and run filebeat
- hosts: filebeat
+- name: Reconfigure and start MISP
+ hosts: mispcontainers
roles:
- - filebeat
+ - misp
- name: Reconfigure and start TheHive
hosts: thehive
diff --git a/stopsoctools.yml b/stopsoctools.yml
index 5bd121746729120486802cfc5538a181235b1642..045c3f06243bc3358245c8bbf219d68e9f89a9a5 100644
--- a/stopsoctools.yml
+++ b/stopsoctools.yml
@@ -6,7 +6,7 @@
- nifi
- name: Stop all containers
- hosts: soctoolsmain
+ hosts: dsldev
roles:
- docker
diff --git a/utils/flow2template.py b/utils/flow2template.py
index b2018328c6ccf5d7c53bf75645c3f44122cc4ba2..570a1ef23d0606d7ef2db44b7133d5a6ec9f43b2 100755
--- a/utils/flow2template.py
+++ b/utils/flow2template.py
@@ -13,20 +13,17 @@ et = xml.etree.ElementTree.parse(f)
for v in et.findall(".//variable"):
a=v.attrib
if a['name']=="misp_token":
- a['value']="{{lookup('file','{{playbook_dir}}/secrets/tokens/misp')}}"
+ a['value']="{{ misp_token }}"
elif a['name']=="misp_url":
a['value']="{{ misp_url }}"
elif a['name']=="maxmind_key":
a['value']="{{ maxmind_key }}"
elif a['name']=="elastic_url":
- a['value']="https://{{ soctoolsproxy }}:9200"
+ a['value']="https://{{ dslproxy }}:9200"
elif a['name']=="elastic_username":
a['value']="{{ elastic_username }}"
elif a['name']=="elastic_password":
- a['value']="{{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}"
-
-for v in et.findall(".//controllerService[name='Soctools CA']/property[name='Truststore Password']/value"):
- v.text="{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
+ a['value']="{{ odfees_adminpass }}"
et.write(args.templatefile)