diff --git a/group_vars/all/main.yml b/group_vars/all/main.yml
index 6fdd65d41e51e49d7171b71ec463d88bfb3eed96..f44018388f7bbdc2f2d81890a5b2bf6b427bceae 100644
--- a/group_vars/all/main.yml
+++ b/group_vars/all/main.yml
@@ -4,7 +4,7 @@ soctools_netname: "dslnifinet"
repo: gn43-dsl
version: 7
-suffix: al
+suffix: a20200516
temp_root: "/tmp/centosbuild"
@@ -35,4 +35,11 @@ ca_cn: "dsldev test ca"
nifiadmin:
- [ "Bozidar Proevski", "Pass001" ]
- [ "Arne Oslebo", "Pass002" ]
+ - [ "NifiELKuser", "Pass003" ]
+odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}"
+odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}"
+
+odfees_adminpass: "Pass004"
+openid_realm: "GN43WP8T31SOC1"
+openid_subjkey: preferred_username
diff --git a/roles/build/tasks/main.yml b/roles/build/tasks/main.yml
index f94d4167a5c7b5bdcee96428d7bacd7496164182..643f91f1a674d0d782fd3d75e381d1acc25ba003 100644
--- a/roles/build/tasks/main.yml
+++ b/roles/build/tasks/main.yml
@@ -6,3 +6,5 @@
- include: openjdk.yml
- include: zookeeper.yml
- include: nifi.yml
+- include: odfees.yml
+- include: odfekibana.yml
diff --git a/roles/build/tasks/odfees.yml b/roles/build/tasks/odfees.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5741223fdd61d30801eec2fc4c2bdbe1fdb7ed36
--- /dev/null
+++ b/roles/build/tasks/odfees.yml
@@ -0,0 +1,18 @@
+---
+
+- name: Configure elasticsearch Dockerfile
+ template:
+ src: odfees/Dockerfile-elastic.j2
+ dest: "{{role_path}}/files/elasticDockerfile"
+
+- name: Build elasticsearch image
+ command: docker build -t {{repo}}/elasticsearch:{{version}}{{suffix}} -f {{role_path}}/files/elasticDockerfile {{role_path}}/files
+
+- name: Configure odfe elasticsearch Dockerfile
+ template:
+ src: odfees/Dockerfile-odfeelastic.j2
+ dest: "{{role_path}}/files/odfeesDockerfile"
+
+- name: Build odfe elasticsearch image
+ command: docker build -t {{repo}}/odfees:{{version}}{{suffix}} -f {{role_path}}/files/odfeesDockerfile {{role_path}}/files
+
diff --git a/roles/build/tasks/odfekibana.yml b/roles/build/tasks/odfekibana.yml
new file mode 100644
index 0000000000000000000000000000000000000000..8ca53578baceacba76726bece7c1dfbe7491c917
--- /dev/null
+++ b/roles/build/tasks/odfekibana.yml
@@ -0,0 +1,19 @@
+---
+
+- name: Configure kibana Dockerfile
+ template:
+ src: odfekibana/Dockerfile-kibana.j2
+ dest: "{{role_path}}/files/kibanaDockerfile"
+
+- name: Build kibana image
+ command: docker build -t {{repo}}/kibana:{{version}}{{suffix}} -f {{role_path}}/files/kibanaDockerfile {{role_path}}/files
+
+- name: Configure odfe kibana Dockerfile
+ template:
+ src: odfekibana/Dockerfile-odfekibana.j2
+ dest: "{{role_path}}/files/odfekibanaDockerfile"
+
+- name: Build odfe kibana image
+ command: docker build -t {{repo}}/odfekibana:{{version}}{{suffix}} -f {{role_path}}/files/odfekibanaDockerfile {{role_path}}/files
+
+
diff --git a/roles/build/templates/nifi/Dockerfile.j2 b/roles/build/templates/nifi/Dockerfile.j2
index f7a48b9f2a48edbe5ca46b4176750abdf67c339d..2c0b69477fe0f5024070e7321e785d2fe561b5d4 100644
--- a/roles/build/templates/nifi/Dockerfile.j2
+++ b/roles/build/templates/nifi/Dockerfile.j2
@@ -22,7 +22,7 @@ FROM {{repo}}/openjdk:{{version}}{{suffix}}
ARG UID=1000
ARG GID=1000
-ARG NIFI_VERSION=1.9.2
+ARG NIFI_VERSION=1.11.4
ARG BASE_URL=https://archive.apache.org/dist
ARG MIRROR_BASE_URL=${MIRROR_BASE_URL:-${BASE_URL}}
ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.zip}
diff --git a/roles/build/templates/odfees/Dockerfile-elastic.j2 b/roles/build/templates/odfees/Dockerfile-elastic.j2
new file mode 100644
index 0000000000000000000000000000000000000000..8ec389d19a74dd53d6e24c679ad3c621985c641b
--- /dev/null
+++ b/roles/build/templates/odfees/Dockerfile-elastic.j2
@@ -0,0 +1,19 @@
+FROM {{repo}}/openjdk:{{version}}{{suffix}}
+
+ENV PATH="/usr/share/elasticsearch/bin:${PATH}"
+
+RUN groupadd -g 1000 elasticsearch && \
+ adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
+
+WORKDIR /usr/share/elasticsearch
+
+RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
+ rpm -Uvh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.6.1-no-jdk-x86_64.rpm && \
+ cp -a /etc/elasticsearch/ /usr/share/elasticsearch/config/ && \
+ chown -R elasticsearch /usr/share/elasticsearch/config && \
+ sed -i -e 's,ES_PATH_CONF=/etc/elasticsearch,ES_PATH_CONF=/usr/share/elasticsearch/config,g' /etc/sysconfig/elasticsearch
+
+EXPOSE 9200 9300
+USER elasticsearch
+ENTRYPOINT ["/bin/bash"]
+
diff --git a/roles/build/templates/odfees/Dockerfile-odfeelastic.j2 b/roles/build/templates/odfees/Dockerfile-odfeelastic.j2
new file mode 100644
index 0000000000000000000000000000000000000000..8fe4cb8620eb5761cdc53d0191fdbfcbfd432edf
--- /dev/null
+++ b/roles/build/templates/odfees/Dockerfile-odfeelastic.j2
@@ -0,0 +1,16 @@
+FROM {{repo}}/elasticsearch:{{version}}{{suffix}}
+
+ENV PATH="/usr/share/elasticsearch/bin:${PATH}"
+
+USER root
+WORKDIR /usr/share/elasticsearch
+
+RUN for PLUGIN in \
+ https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-security/opendistro_security-1.6.0.0.zip \
+ https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-alerting/opendistro_alerting-1.6.0.0.zip \
+ https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-sql/opendistro_sql-1.6.0.0.zip; \
+ do bin/elasticsearch-plugin install -b ${PLUGIN}; done && \
+ chown -R elasticsearch plugins/opendistro_security
+
+USER elasticsearch
+
diff --git a/roles/build/templates/odfekibana/Dockerfile-kibana.j2 b/roles/build/templates/odfekibana/Dockerfile-kibana.j2
new file mode 100644
index 0000000000000000000000000000000000000000..fe136898a514a8773f164f326f704f2ef772a9d9
--- /dev/null
+++ b/roles/build/templates/odfekibana/Dockerfile-kibana.j2
@@ -0,0 +1,18 @@
+FROM {{repo}}/centos:{{version}}{{suffix}}
+
+ENV PATH="/usr/share/kibana/bin:${PATH}"
+
+RUN groupadd -g 1000 kibana && \
+ adduser -u 1000 -g 1000 -d /usr/share/kibana kibana
+
+WORKDIR /usr/share/kibana
+
+RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
+ rpm -Uvh https://artifacts.elastic.co/downloads/kibana/kibana-oss-7.6.1-x86_64.rpm && \
+ cp -a /etc/kibana/ /usr/share/kibana/config/ && \
+ chown -R kibana /usr/share/kibana/config/
+
+EXPOSE 5601
+USER kibana
+ENTRYPOINT ["/bin/bash"]
+
diff --git a/roles/build/templates/odfekibana/Dockerfile-odfekibana.j2 b/roles/build/templates/odfekibana/Dockerfile-odfekibana.j2
new file mode 100644
index 0000000000000000000000000000000000000000..b127fc2229c8d0a301e0be9f33de787468b49dab
--- /dev/null
+++ b/roles/build/templates/odfekibana/Dockerfile-odfekibana.j2
@@ -0,0 +1,15 @@
+FROM {{repo}}/kibana:{{version}}{{suffix}}
+
+ENV PATH="/usr/share/kibana/bin:${PATH}"
+
+USER root
+WORKDIR /usr/share/kibana
+
+RUN for PLUGIN in \
+ https://d3g5vo6xdbdb9a.cloudfront.net/downloads/kibana-plugins/opendistro-security/opendistro_security_kibana_plugin-1.6.0.0.zip \
+ https://d3g5vo6xdbdb9a.cloudfront.net/downloads/kibana-plugins/opendistro-alerting/opendistro-alerting-1.6.0.0.zip \
+ https://d3g5vo6xdbdb9a.cloudfront.net/downloads/kibana-plugins/opendistro-index-management/opendistro_index_management_kibana-1.6.0.0.zip; \
+ do bin/kibana-plugin install --allow-root ${PLUGIN}; done
+
+USER kibana
+
diff --git a/roles/ca/tasks/main.yml b/roles/ca/tasks/main.yml
index 3a5ffcd959aab707804b1d8af145e574e8dba882..a5ed4a4294f623dd9a23b4709dc11fc232c41fb9 100644
--- a/roles/ca/tasks/main.yml
+++ b/roles/ca/tasks/main.yml
@@ -30,7 +30,7 @@
- name: Generate truststore
command: >
docker run --rm -v {{role_path}}/files/truststore/:/opt/cafiles/:z
- gn43-dsl/openjdk:7al keytool -import -noprompt -trustcacerts
+ "{{repo}}/openjdk:{{version}}{{suffix}}" keytool -import -noprompt -trustcacerts
-alias "{{item}}" -file "/opt/cafiles/{{item}}.crt" -keystore /opt/cafiles/cacerts.jks -storepass "{{tspass}}"
with_items:
- "{{ ca_cn }}"
@@ -40,6 +40,8 @@
command: roles/ca/files/easyrsa/easyrsa show-cert {{item}}
with_items:
- "{{ groups['nificontainers'] }}"
+ - "{{ groups['odfeescontainers'] }}"
+ - "{{ groups['odfekibanacontainers'] }}"
environment:
EASYRSA_BATCH: 1
EASYRSA_PKI: roles/ca/files/CA
@@ -53,6 +55,8 @@
build-serverClient-full {{item}} nopass
with_items:
- "{{ groups['nificontainers'] }}"
+ - "{{ groups['odfeescontainers'] }}"
+ - "{{ groups['odfekibanacontainers'] }}"
environment:
EASYRSA_BATCH: 1
EASYRSA_PKI: roles/ca/files/CA
@@ -82,6 +86,8 @@
Enter Export Password: "{{kspass}}"
with_items:
- "{{ groups['nificontainers'] }}"
+ - "{{ groups['odfeescontainers'] }}"
+ - "{{ groups['odfekibanacontainers'] }}"
environment:
EASYRSA_BATCH: 1
EASYRSA_PKI: roles/ca/files/CA
@@ -93,10 +99,42 @@
with_items:
- "{{ groups['nificontainers'] }}"
-- name: Copy nifi truststore to nifi role
+- name: Copy odfees host certs to odfees role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.p12
+ dest: roles/odfees/files/{{item}}.p12
+ with_items:
+ - "{{ groups['odfeescontainers'] }}"
+
+- name: Copy odfekibana host p12 certs to odfekibana role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.p12
+ dest: roles/odfekibana/files/{{item}}.p12
+ with_items:
+ - "{{ groups['odfekibanacontainers'] }}"
+
+- name: Copy odfekibana host certs to odfekibana role
+ copy:
+ src: roles/ca/files/CA/issued/{{item}}.crt
+ dest: roles/odfekibana/files/{{item}}.crt
+ with_items:
+ - "{{ groups['odfekibanacontainers'] }}"
+
+- name: Copy odfekibana host keys to odfekibana role
+ copy:
+ src: roles/ca/files/CA/private/{{item}}.key
+ dest: roles/odfekibana/files/{{item}}.key
+ with_items:
+ - "{{ groups['odfekibanacontainers'] }}"
+
+- name: Copy truststore to roles
copy:
src: roles/ca/files/truststore/cacerts.jks
- dest: roles/nifi/files/cacerts.jks
+ dest: "roles/{{item}}/files/cacerts.jks"
+ with_items:
+ - nifi
+ - odfees
+ - odfekibana
- name: Check for existing user certificates
command: roles/ca/files/easyrsa/easyrsa show-cert {{item[0] | regex_escape()}}
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index fd2400e0ff5bbe66918e71939d02422de2df6e1f..23ced1adceecc87ab1ad3505eb1e34570a47cda8 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -3,6 +3,8 @@
- include: networkcreate.yml
- include: zookeeper.yml
- include: nifi.yml
+- include: odfees.yml
+- include: odfekibana.yml
- include: nginx.yml
- include: networkremove.yml
diff --git a/roles/docker/tasks/nginx.yml b/roles/docker/tasks/nginx.yml
index 3a45bfaa8c97ea377e182571f8ed061d1e98b186..683c6364b6cd1992f64c0eeeb50daca632c233d0 100644
--- a/roles/docker/tasks/nginx.yml
+++ b/roles/docker/tasks/nginx.yml
@@ -10,6 +10,9 @@
networks_cli_compatible: yes
published_ports:
- "443:443"
+ - "8443:8443"
+ - "9443:9443"
+ - "9200:9200"
- "7750:7750"
tags:
- start
diff --git a/roles/docker/tasks/odfees.yml b/roles/docker/tasks/odfees.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a159eadc3382254b6877bfdd32acc5c4c238d29e
--- /dev/null
+++ b/roles/docker/tasks/odfees.yml
@@ -0,0 +1,24 @@
+---
+
+- name: Create odfe elasticsearch containers and connect to network
+ docker_container:
+ name: "{{ item }}"
+ hostname: "{{ item }}"
+ image: "{{ odfees_img }}"
+ networks:
+ - name: "{{ soctools_netname }}"
+ networks_cli_compatible: yes
+ entrypoint: "/bin/bash"
+ interactive: "yes"
+ with_items: "{{ groups['odfeescontainers'] }}"
+ tags:
+ - start
+
+- name: Disconnect odfe elasticsearch containers from network and remove
+ docker_container:
+ name: "{{ item }}"
+ state: absent
+ with_items: "{{ groups['odfeescontainers'] }}"
+ tags:
+ - stop
+
diff --git a/roles/docker/tasks/odfekibana.yml b/roles/docker/tasks/odfekibana.yml
new file mode 100644
index 0000000000000000000000000000000000000000..33feb906294eeda07eaae6ea2fa001c0a9b91a8b
--- /dev/null
+++ b/roles/docker/tasks/odfekibana.yml
@@ -0,0 +1,26 @@
+---
+
+- name: Create odfe kibana containers and connect to network
+ docker_container:
+ name: "{{ item }}"
+ hostname: "{{ item }}"
+ image: "{{ odfekibana_img }}"
+ networks:
+ - name: "{{ soctools_netname }}"
+ networks_cli_compatible: yes
+ published_ports:
+ - "5601:5601"
+ entrypoint: "/bin/bash"
+ interactive: "yes"
+ with_items: "{{ groups['odfekibanacontainers'] }}"
+ tags:
+ - start
+
+- name: Disconnect odfe elasticsearch containers from network and remove
+ docker_container:
+ name: "{{ item }}"
+ state: absent
+ with_items: "{{ groups['odfekibanacontainers'] }}"
+ tags:
+ - stop
+
diff --git a/roles/nginx/templates/nginx.conf.j2 b/roles/nginx/templates/nginx.conf.j2
index 25a191866c96bfc839b5fe724dc064599caa6da5..69614b2bb47ec777823a6f3028bcd966510a25bf 100644
--- a/roles/nginx/templates/nginx.conf.j2
+++ b/roles/nginx/templates/nginx.conf.j2
@@ -12,15 +12,26 @@ stream {
resolver 127.0.0.11;
upstream nifiserv {
+ hash $remote_addr consistent;
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}}:9443;
{% endfor %}
}
server {
- listen 443;
+ listen 9443;
proxy_pass nifiserv;
}
+ upstream odfeserv {
+ {% for odfehost in groups['odfeescontainers'] %}
+ server {{odfehost}}:9200;
+ {% endfor %}
+ }
+ server {
+ listen 9200;
+ proxy_pass odfeserv;
+ }
+
upstream nifiservtcp7750 {
{% for nifihost in groups['nificontainers'] %}
server {{nifihost}}:7750;
diff --git a/roles/nifi/templates/nifi.properties.j2 b/roles/nifi/templates/nifi.properties.j2
index e655879e0e3b6e7ea94d047ddcf5e6706c821892..6830450c0e433a2cc69ee73eb06ca68a53a7d146 100644
--- a/roles/nifi/templates/nifi.properties.j2
+++ b/roles/nifi/templates/nifi.properties.j2
@@ -143,7 +143,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
nifi.web.max.header.size=16 KB
nifi.web.proxy.context.path=/nifi
-nifi.web.proxy.host={{ dslproxy }}
+nifi.web.proxy.host={{ dslproxy }}:9443
# security properties #
nifi.sensitive.props.key=
diff --git a/roles/odfees/defaults/main.yml b/roles/odfees/defaults/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfees/files/.empty b/roles/odfees/files/.empty
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfees/handlers/main.yml b/roles/odfees/handlers/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfees/meta/main.yml b/roles/odfees/meta/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfees/tasks/main.yml b/roles/odfees/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e05ae27e16be19e0443e443b8650e73355a82d06
--- /dev/null
+++ b/roles/odfees/tasks/main.yml
@@ -0,0 +1,115 @@
+---
+
+- name: Create config directory
+ file:
+ name: config
+ state: directory
+ mode: 0700
+ tags:
+ - start
+
+- name: Copy certificates in odfe conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "config/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.p12"
+ - cacerts.jks
+ - "{{nifiadmin.0[0]}}.p12"
+ tags:
+ - start
+
+- name: Configure sysconfig
+ template:
+ src: sysconfig_elasticsearch.j2
+ dest: sysconfig_elasticsearch
+ tags:
+ - start
+
+- name: Copy sysconfig to /etc
+ command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
+ tags:
+ - start
+
+# lineinfile:
+# path: /etc/sysconfig/elasticsearch
+# regexp: '^ES_PATH_CONF='
+# line: ES_PATH_CONF=/usr/share/elasticsearch/config
+# tags:
+# - start
+- name: Configure odfe properties
+ template:
+ src: "config/{{item}}.j2"
+ dest: "config/{{item}}"
+ with_items:
+ - elasticsearch.yml
+ - jvm.options
+ - log4j2.properties
+ tags:
+ - start
+
+- name: Change password for admin
+ command: "bash plugins/opendistro_security/tools/hash.sh -p {{odfees_adminpass}}"
+ register: adminhash
+ # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
+ tags:
+ - start
+
+- set_fact:
+ adminhashpwd: "{{ adminhash.stdout }}"
+ #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}"
+ tags:
+ - start
+
+- name: Configure opendistro_security properties
+ template:
+ src: "securityconfig/{{item}}.j2"
+ dest: "plugins/opendistro_security/securityconfig/{{item}}"
+ #dest: "{{item}}"
+ with_items:
+ - internal_users.yml
+ - config.yml
+ tags:
+ - start
+
+#- name: Exit here to test ODFE
+# meta: end_play
+# tags:
+# - start
+
+- name: Start OpenDistro for Elasticsearch
+ command: "/usr/share/elasticsearch/bin/elasticsearch -p {{ inventory_hostname }}.pid -d"
+ tags:
+ - start
+
+- name: Wait for ElasticSearch
+ wait_for:
+ host: "{{groups['odfeescontainers'][0]}}"
+ port: 9200
+ state: started
+ delay: 5
+ tags:
+ - start
+
+- name: Configure OpenDistro security
+ command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{nifiadmin.0[0]}}.p12' -kspass {{nifiadmin.0[1]}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{tspass}} -cn dsoclab-cluster"
+ when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
+ tags:
+ - start
+
+#- name: check reachable hosts
+# gather_facts: no
+# tasks:
+# - command: ping -c1 {{ inventory_hostname }}
+# delegate_to: localhost
+# register: ping_result
+# ignore_errors: yes
+# - group_by: key=reachable
+# when: ping_result|success
+
+- name: Stop OpenDistro for Elasticsearch
+ command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+ tags:
+ - stop
+
diff --git a/roles/odfees/templates/config/elasticsearch.yml.j2 b/roles/odfees/templates/config/elasticsearch.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..928b1ed9f9a75620a0a8c0212ab56eea862998d7
--- /dev/null
+++ b/roles/odfees/templates/config/elasticsearch.yml.j2
@@ -0,0 +1,67 @@
+cluster.name: "dsoclab-cluster"
+#network.host: 0.0.0.0
+network.host: {{ inventory_hostname }}
+discovery.seed_hosts:
+{% for odfees in groups['odfeescontainers'] %}
+ - {{ odfees }}
+{% endfor %}
+#discovery.type: single-node
+transport.port: 9300
+
+path.logs: /usr/share/elasticsearch/logs
+# # minimum_master_nodes need to be explicitly set when bound on a public IP
+# # set to 1 to allow single node clusters
+# # Details: https://github.com/elastic/elasticsearch/pull/17288
+#discovery.zen.minimum_master_nodes: 1
+
+# # Breaking change in 7.0
+# # https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#breaking_70_discovery_changes
+cluster.initial_master_nodes:
+{% for odfees in groups['odfeescontainers'] %}
+ - {{ odfees }}
+{% endfor %}
+
+# - elasticsearch1
+# - docker-test-node-1
+######## Start OpenDistro for Elasticsearch Security Demo Configuration ########
+# WARNING: revise all the lines below before you go into production
+# opendistro_security.ssl.transport.pemcert_filepath: esnode.pem
+# opendistro_security.ssl.transport.pemkey_filepath: esnode-key.pem
+
+opendistro_security.ssl.transport.keystore_type: pkcs12
+opendistro_security.ssl.transport.keystore_filepath: {{ inventory_hostname }}.p12
+opendistro_security.ssl.transport.keystore_password: {{ kspass }}
+#opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
+opendistro_security.ssl.transport.truststore_type: jks
+opendistro_security.ssl.transport.truststore_filepath: cacerts.jks
+opendistro_security.ssl.transport.truststore_password: {{ tspass }}
+opendistro_security.ssl.transport.enforce_hostname_verification: false
+
+opendistro_security.ssl.http.enabled: true
+# opendistro_security.ssl.http.pemcert_filepath: esnode.pem
+# opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem
+opendistro_security.ssl.http.keystore_type: pkcs12
+opendistro_security.ssl.http.keystore_filepath: {{ inventory_hostname }}.p12
+opendistro_security.ssl.http.keystore_password: {{ kspass }}
+opendistro_security.ssl.http.truststore_type: jks
+opendistro_security.ssl.http.truststore_filepath: cacerts.jks
+opendistro_security.ssl.http.truststore_password: {{ tspass }}
+#opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem
+#opendistro_security.ssl.http.clientauth_mode: optional
+opendistro_security.allow_unsafe_democertificates: false
+opendistro_security.allow_default_init_securityindex: false
+opendistro_security.authcz.admin_dn:
+ - "CN={{ nifiadmin[0][0] }}"
+
+opendistro_security.nodes_dn:
+{% for odfees in groups['odfeescontainers'] %}
+ - "CN={{ odfees }}"
+{% endfor %}
+
+opendistro_security.audit.type: internal_elasticsearch
+opendistro_security.enable_snapshot_restore_privilege: true
+opendistro_security.check_snapshot_restore_write_privileges: true
+opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
+cluster.routing.allocation.disk.threshold_enabled: false
+node.max_local_storage_nodes: 3
+######## End OpenDistro for Elasticsearch Security Demo Configuration ########
diff --git a/roles/odfees/templates/config/jvm.options.j2 b/roles/odfees/templates/config/jvm.options.j2
new file mode 100644
index 0000000000000000000000000000000000000000..8717fc8f40604981baeffa02970ae213145f78e5
--- /dev/null
+++ b/roles/odfees/templates/config/jvm.options.j2
@@ -0,0 +1,119 @@
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms512m
+-Xmx512m
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1GC is only supported on JDK version 10 or later.
+# To use G1GC uncomment the lines below.
+# 10-:-XX:-UseConcMarkSweepGC
+# 10-:-XX:-UseCMSInitiatingOccupancyOnly
+# 10-:-XX:+UseG1GC
+# 10-:-XX:InitiatingHeapOccupancyPercent=75
+
+## DNS cache policy
+# cache ttl in seconds for positive DNS lookups noting that this overrides the
+# JDK security property networkaddress.cache.ttl; set to -1 to cache forever
+-Des.networkaddress.cache.ttl=60
+# cache ttl in seconds for negative DNS lookups noting that this overrides the
+# JDK security property networkaddress.cache.negative ttl; set to -1 to cache
+# forever
+-Des.networkaddress.cache.negative.ttl=10
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# explicitly set the stack size
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# turn off a JDK optimization that throws away stack traces for common
+# exceptions because stack traces are important for debugging
+-XX:-OmitStackTraceInFastThrow
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=data
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=logs/hs_err_pid%p.log
+
+## JDK 8 GC logging
+
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:logs/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
+# time/date parsing will break in an incompatible way for some date patterns and locals
+9-:-Djava.locale.providers=COMPAT
diff --git a/roles/odfees/templates/config/log4j2.properties.j2 b/roles/odfees/templates/config/log4j2.properties.j2
new file mode 100644
index 0000000000000000000000000000000000000000..9ad290ad82679309319cee88bee3eaf9d49814eb
--- /dev/null
+++ b/roles/odfees/templates/config/log4j2.properties.j2
@@ -0,0 +1,9 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
diff --git a/roles/odfees/templates/securityconfig/action_groups.yml b/roles/odfees/templates/securityconfig/action_groups.yml
new file mode 100644
index 0000000000000000000000000000000000000000..aac69ab8f804fc320dc4c58529f93f5ad8aa7601
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/action_groups.yml
@@ -0,0 +1,3 @@
+_meta:
+ type: "actiongroups"
+ config_version: 2
\ No newline at end of file
diff --git a/roles/odfees/templates/securityconfig/config.yml.j2 b/roles/odfees/templates/securityconfig/config.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..ccaf0a2c7902ead40b628b031c30be7e0e628f8c
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/config.yml.j2
@@ -0,0 +1,263 @@
+---
+
+# This is the main Open Distro Security configuration file where authentication
+# and authorization is defined.
+#
+# You need to configure at least one authentication domain in the authc of this file.
+# An authentication domain is responsible for extracting the user credentials from
+# the request and for validating them against an authentication backend like Active Directory for example.
+#
+# If more than one authentication domain is configured the first one which succeeds wins.
+# If all authentication domains fail then the request is unauthenticated.
+# In this case an exception is thrown and/or the HTTP status is set to 401.
+#
+# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect
+# the roles from a given backend for the authenticated user.
+#
+# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both.
+# http_enabled: true
+# transport_enabled: true
+#
+# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to
+# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated.
+# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous"
+# and one role named "anonymous_backendrole".
+# If you enable anonymous authentication all HTTP authenticators will not challenge.
+#
+#
+# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert"
+# first and the challenging one last.
+# Because it's not possible to challenge a client with two different authentication methods (for example
+# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation
+# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request.
+#
+# Default value of the challenge flag is true.
+#
+#
+# HTTP
+# basic (challenging)
+# proxy (not challenging, needs xff)
+# kerberos (challenging)
+# clientcert (not challenging, needs https)
+# jwt (not challenging)
+# host (not challenging) #DEPRECATED, will be removed in a future version.
+# host based authentication is configurable in roles_mapping
+
+# Authc
+# internal
+# noop
+# ldap
+
+# Authz
+# ldap
+# noop
+
+
+
+_meta:
+ type: "config"
+ config_version: 2
+
+config:
+ dynamic:
+ # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index
+ # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default)
+ # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently
+ #filtered_alias_mode: warn
+ #do_not_fail_on_forbidden: false
+ #kibana:
+ # Kibana multitenancy
+ #multitenancy_enabled: true
+ #server_username: kibanaserver
+ #index: '.kibana'
+ http:
+ anonymous_auth_enabled: false
+ xff:
+ enabled: false
+ internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern
+ #internalProxies: '.*' # trust all internal proxies, regex pattern
+ #remoteIpHeader: 'x-forwarded-for'
+ ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help
+ ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For
+ ###### and here https://tools.ietf.org/html/rfc7239
+ ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve
+ authc:
+ kerberos_auth_domain:
+ http_enabled: false
+ transport_enabled: false
+ order: 6
+ http_authenticator:
+ type: kerberos
+ challenge: true
+ config:
+ # If true a lot of kerberos/security related debugging output will be logged to standard out
+ krb_debug: false
+ # If true then the realm will be stripped from the user name
+ strip_realm_from_principal: true
+ authentication_backend:
+ type: noop
+ basic_internal_auth_domain:
+ description: "Authenticate via HTTP Basic against internal users database"
+ http_enabled: true
+ transport_enabled: true
+ order: 4
+ http_authenticator:
+ type: basic
+ challenge: true
+ authentication_backend:
+ type: intern
+ openid_auth_domain:
+ http_enabled: true
+ transport_enabled: true
+ order: 1
+ http_authenticator:
+ type: openid
+ challenge: false
+ config:
+ subject_key: {{openid_subjkey}}
+ roles_key: roles
+ openid_connect_url: https://{{dslproxy}}:10443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
+ enable_ssl: true
+ verify_hostnames: false
+ # pemtrustedcas_filepath: /usr/share/elasticsearch/config/dslca.crt
+ authentication_backend:
+ type: noop
+ proxy_auth_domain:
+ description: "Authenticate via proxy"
+ http_enabled: false
+ transport_enabled: false
+ order: 3
+ http_authenticator:
+ type: proxy
+ challenge: false
+ config:
+ user_header: "x-proxy-user"
+ roles_header: "x-proxy-roles"
+ authentication_backend:
+ type: noop
+ jwt_auth_domain:
+ description: "Authenticate via Json Web Token"
+ http_enabled: false
+ transport_enabled: false
+ order: 0
+ http_authenticator:
+ type: jwt
+ challenge: false
+ config:
+ signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key"
+ jwt_header: "Authorization"
+ jwt_url_parameter: null
+ roles_key: null
+ subject_key: null
+ authentication_backend:
+ type: noop
+ clientcert_auth_domain:
+ description: "Authenticate via SSL client certificates"
+ http_enabled: false
+ transport_enabled: false
+ order: 2
+ http_authenticator:
+ type: clientcert
+ config:
+ username_attribute: cn #optional, if omitted DN becomes username
+ challenge: false
+ authentication_backend:
+ type: noop
+ ldap:
+ description: "Authenticate via LDAP or Active Directory"
+ http_enabled: false
+ transport_enabled: false
+ order: 5
+ http_authenticator:
+ type: basic
+ challenge: false
+ authentication_backend:
+ # LDAP authentication backend (authenticate users against a LDAP or Active Directory)
+ type: ldap
+ config:
+ # enable ldaps
+ enable_ssl: false
+ # enable start tls, enable_ssl should be false
+ enable_start_tls: false
+ # send client certificate
+ enable_ssl_client_auth: false
+ # verify ldap hostname
+ verify_hostnames: true
+ hosts:
+ - localhost:8389
+ bind_dn: null
+ password: null
+ userbase: 'ou=people,dc=example,dc=com'
+ # Filter to search for users (currently in the whole subtree beneath userbase)
+ # {0} is substituted with the username
+ usersearch: '(sAMAccountName={0})'
+ # Use this attribute from the user as username (if not set then DN is used)
+ username_attribute: null
+ authz:
+ roles_from_myldap:
+ description: "Authorize via LDAP or Active Directory"
+ http_enabled: false
+ transport_enabled: false
+ authorization_backend:
+ # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too)
+ type: ldap
+ config:
+ # enable ldaps
+ enable_ssl: false
+ # enable start tls, enable_ssl should be false
+ enable_start_tls: false
+ # send client certificate
+ enable_ssl_client_auth: false
+ # verify ldap hostname
+ verify_hostnames: true
+ hosts:
+ - localhost:8389
+ bind_dn: null
+ password: null
+ rolebase: 'ou=groups,dc=example,dc=com'
+ # Filter to search for roles (currently in the whole subtree beneath rolebase)
+ # {0} is substituted with the DN of the user
+ # {1} is substituted with the username
+ # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute
+ rolesearch: '(member={0})'
+ # Specify the name of the attribute which value should be substituted with {2} above
+ userroleattribute: null
+ # Roles as an attribute of the user entry
+ userrolename: disabled
+ #userrolename: memberOf
+ # The attribute in a role entry containing the name of that role, Default is "name".
+ # Can also be "dn" to use the full DN as rolename.
+ rolename: cn
+ # Resolve nested roles transitive (roles which are members of other roles and so on ...)
+ resolve_nested_roles: true
+ userbase: 'ou=people,dc=example,dc=com'
+ # Filter to search for users (currently in the whole subtree beneath userbase)
+ # {0} is substituted with the username
+ usersearch: '(uid={0})'
+ # Skip users matching a user name, a wildcard or a regex pattern
+ #skip_users:
+ # - 'cn=Michael Jackson,ou*people,o=TEST'
+ # - '/\S*/'
+ roles_from_another_ldap:
+ description: "Authorize via another Active Directory"
+ http_enabled: false
+ transport_enabled: false
+ authorization_backend:
+ type: ldap
+ #config goes here ...
+ # auth_failure_listeners:
+ # ip_rate_limiting:
+ # type: ip
+ # allowed_tries: 10
+ # time_window_seconds: 3600
+ # block_expiry_seconds: 600
+ # max_blocked_clients: 100000
+ # max_tracked_clients: 100000
+ # internal_authentication_backend_limiting:
+ # type: username
+ # authentication_backend: intern
+ # allowed_tries: 10
+ # time_window_seconds: 3600
+ # block_expiry_seconds: 600
+ # max_blocked_clients: 100000
+ # max_tracked_clients: 100000
diff --git a/roles/odfees/templates/securityconfig/elasticsearch.yml.example b/roles/odfees/templates/securityconfig/elasticsearch.yml.example
new file mode 100644
index 0000000000000000000000000000000000000000..9151edf3abe2f29b13547cabef2869ee3267a3e4
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/elasticsearch.yml.example
@@ -0,0 +1,209 @@
+############## Open Distro Security configuration ###############
+
+###########################################################
+# Add the following settings to your standard elasticsearch.yml
+# alongside with the Open Distro Security TLS settings.
+# Settings must always be the same on all nodes in the cluster.
+
+############## Common configuration settings ##############
+
+# Enable or disable the Open Distro Security advanced modules
+# By default advanced modules are enabled, you can switch
+# all advanced features off by setting the following key to false
+opendistro_security.advanced_modules_enabled: true
+
+# Specify a list of DNs which denote the other nodes in the cluster.
+# This settings support wildcards and regular expressions
+# This setting only has effect if 'opendistro_security.cert.intercluster_request_evaluator_class' is not set.
+opendistro_security.nodes_dn:
+ - "CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE"
+ - "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE"
+
+# Defines the DNs (distinguished names) of certificates
+# to which admin privileges should be assigned (mandatory)
+opendistro_security.authcz.admin_dn:
+ - "CN=kirk,OU=client,O=client,l=tEst, C=De"
+
+# Define how backend roles should be mapped to Open Distro Security roles
+# MAPPING_ONLY - mappings must be configured explicitely in roles_mapping.yml (default)
+# BACKENDROLES_ONLY - backend roles are mapped to Open Distro Security rules directly. Settings in roles_mapping.yml have no effect.
+# BOTH - backend roles are mapped to Open Distro Security roles mapped directly and via roles_mapping.yml in addition
+opendistro_security.roles_mapping_resolution: MAPPING_ONLY
+
+############## REST Management API configuration settings ##############
+# Enable or disable role based access to the REST management API
+# Default is that no role is allowed to access the REST management API.
+#opendistro_security.restapi.roles_enabled: ["all_access","xyz_role"]
+
+# Disable particular endpoints and their HTTP methods for roles.
+# By default all endpoints/methods are allowed.
+#opendistro_security.restapi.endpoints_disabled.<role>.<endpoint>: <array of http methods>
+# Example:
+#opendistro_security.restapi.endpoints_disabled.all_access.ACTIONGROUPS: ["PUT","POST","DELETE"]
+#opendistro_security.restapi.endpoints_disabled.xyz_role.LICENSE: ["DELETE"]
+
+# The following endpoints exist:
+# ACTIONGROUPS
+# CACHE
+# CONFIG
+# ROLES
+# ROLESMAPPING
+# INTERNALUSERS
+# SYSTEMINFO
+# PERMISSIONSINFO
+
+############## Auditlog configuration settings ##############
+# General settings
+
+# Enable/disable rest request logging (default: true)
+#opendistro_security.audit.enable_rest: true
+# Enable/disable transport request logging (default: false)
+#opendistro_security.audit.enable_transport: false
+# Enable/disable bulk request logging (default: false)
+# If enabled all subrequests in bulk requests will be logged too
+#opendistro_security.audit.resolve_bulk_requests: false
+# Disable some categories
+#opendistro_security.audit.config.disabled_categories: ["AUTHENTICATED","GRANTED_PRIVILEGES"]
+# Disable some requests (wildcard or regex of actions or rest request paths)
+#opendistro_security.audit.ignore_requests: ["indices:data/read/*","*_bulk"]
+# Tune threadpool size, default is 10 and 0 means disabled
+#opendistro_security.audit.threadpool.size: 0
+# Tune threadpool max size queue length, default is 100000
+#opendistro_security.audit.threadpool.max_queue_len: 100000
+
+# If enable_request_details is true then the audit log event will also contain
+# details like the search query. Default is false.
+#opendistro_security.audit.enable_request_details: true
+# Ignore users, e.g. do not log audit requests from that users (default: no ignored users)
+#opendistro_security.audit.ignore_users: ['kibanaserver','some*user','/also.*regex possible/']"
+
+# Destination of the auditlog events
+opendistro_security.audit.type: internal_elasticsearch
+#opendistro_security.audit.type: external_elasticsearch
+#opendistro_security.audit.type: debug
+#opendistro_security.audit.type: webhook
+
+# external_elasticsearch settings
+#opendistro_security.audit.config.http_endpoints: ['localhost:9200','localhost:9201','localhost:9202']"
+# Auditlog index can be a static one or one with a date pattern (default is 'auditlog6')
+#opendistro_security.audit.config.index: auditlog6 # make sure you secure this index properly
+#opendistro_security.audit.config.index: "'auditlog6-'YYYY.MM.dd" #rotates index daily - make sure you secure this index properly
+#opendistro_security.audit.config.type: auditlog
+#opendistro_security.audit.config.username: auditloguser
+#opendistro_security.audit.config.password: auditlogpassword
+#opendistro_security.audit.config.enable_ssl: false
+#opendistro_security.audit.config.verify_hostnames: false
+#opendistro_security.audit.config.enable_ssl_client_auth: false
+#opendistro_security.audit.config.cert_alias: mycert
+#opendistro_security.audit.config.pemkey_filepath: key.pem
+#opendistro_security.audit.config.pemkey_content: <...pem base 64 content>
+#opendistro_security.audit.config.pemkey_password: secret
+#opendistro_security.audit.config.pemcert_filepath: cert.pem
+#opendistro_security.audit.config.pemcert_content: <...pem base 64 content>
+#opendistro_security.audit.config.pemtrustedcas_filepath: ca.pem
+#opendistro_security.audit.config.pemtrustedcas_content: <...pem base 64 content>
+
+# webhook settings
+#opendistro_security.audit.config.webhook.url: "http://mywebhook/endpoint"
+# One of URL_PARAMETER_GET,URL_PARAMETER_POST,TEXT,JSON,SLACK
+#opendistro_security.audit.config.webhook.format: JSON
+#opendistro_security.audit.config.webhook.ssl.verify: false
+#opendistro_security.audit.config.webhook.ssl.pemtrustedcas_filepath: ca.pem
+#opendistro_security.audit.config.webhook.ssl.pemtrustedcas_content: <...pem base 64 content>
+
+# log4j settings
+#opendistro_security.audit.config.log4j.logger_name: auditlogger
+#opendistro_security.audit.config.log4j.level: INFO
+
+############## Kerberos configuration settings ##############
+# If Kerberos authentication should be used you have to configure:
+
+# The Path to the krb5.conf file
+# Can be absolute or relative to the Elasticsearch config directory
+#opendistro_security.kerberos.krb5_filepath: '/etc/krb5.conf'
+
+# The Path to the keytab where the acceptor_principal credentials are stored.
+# Must be relative to the Elasticsearch config directory
+#opendistro_security.kerberos.acceptor_keytab_filepath: 'eskeytab.tab'
+
+# Acceptor (Server) Principal name, must be present in acceptor_keytab_path file
+#opendistro_security.kerberos.acceptor_principal: 'HTTP/localhost'
+
+############## Advanced configuration settings ##############
+# Enable transport layer impersonation
+# Allow DNs (distinguished names) to impersonate as other users
+#opendistro_security.authcz.impersonation_dn:
+# "CN=spock,OU=client,O=client,L=Test,C=DE":
+# - worf
+# "cn=webuser,ou=IT,ou=IT,dc=company,dc=com":
+# - user2
+# - user1
+
+# Enable rest layer impersonation
+# Allow users to impersonate as other users
+#opendistro_security.authcz.rest_impersonation_user:
+# "picard":
+# - worf
+# "john":
+# - steve
+# - martin
+
+# If this is set to true Open Distro Security will automatically initialize the configuration index
+# with the files in the config directory if the index does not exist.
+# WARNING: This will use well-known default passwords.
+# Use only in a private network/environment.
+#opendistro_security.allow_default_init_securityindex: false
+
+# If this is set to true then allow to startup with demo certificates.
+# These are certificates issued by floragunn GmbH for demo purposes.
+# WARNING: This certificates are well known and therefore unsafe
+# Use only in a private network/environment.
+#opendistro_security.allow_unsafe_democertificates: false
+
+############## Expert settings ##############
+# WARNING: Expert settings, do only use if you know what you are doing
+# If you set wrong values here this this could be a security risk
+# or make Open Distro Security stop working
+
+# Name of the index where .opendistro_security stores its configuration.
+
+#opendistro_security.config_index_name: .opendistro_security
+
+# This defines the OID of server node certificates
+#opendistro_security.cert.oid: '1.2.3.4.5.5'
+
+# This specifies the implementation of com.amazon.opendistroforelasticsearch.security.transport.InterClusterRequestEvaluator
+# that is used to determine inter-cluster request.
+# Instances of com.amazon.opendistroforelasticsearch.security.transport.InterClusterRequestEvaluator must implement a single argument
+# constructor that takes an org.elasticsearch.common.settings.Settings
+#opendistro_security.cert.intercluster_request_evaluator_class: com.amazon.opendistroforelasticsearch.security.transport.DefaultInterClusterRequestEvaluator
+
+# Allow snapshot restore for normal users
+# By default only requests signed by an admin TLS certificate can do this
+# To enable snapshot restore for normal users set 'opendistro_security.enable_snapshot_restore_privilege: true'
+# The user who wants to restore a snapshot must have the 'cluster:admin/snapshot/restore' privilege and must also have
+# "indices:admin/create" and "indices:data/write/index" for the indices to be restores.
+# A snapshot can only be restored when it does not contain global state and does not restore the '.opendistro_security' index
+# If 'opendistro_security.check_snapshot_restore_write_privileges: false' is set then the additional indices checks are omitted.
+
+# This makes it less secure.
+#opendistro_security.enable_snapshot_restore_privilege: true
+#opendistro_security.check_snapshot_restore_write_privileges: false
+
+# Authentication cache timeout in minutes (A value of 0 disables caching, default is 60)
+#opendistro_security.cache.ttl_minutes: 60
+
+# Disable Open Distro Security
+# WARNING: This can expose your configuration (including passwords) to the public.
+#opendistro_security.disabled: false
+
+
+# Protected indices are even more secure than normal indices. These indices require a role to access like any other index, but they require an additional role
+# to be visible, listed in the opendistro_security.protected_indices.roles setting.
+# Enable protected indices
+# opendistro_security.protected_indices.enabled: true
+# Specify a list of roles a user must be member of to touch any protected index.
+# opendistro_security.protected_indices.roles: ['all_access']
+# Specify a list of indices to mark as protected. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role.
+# opendistro_security.protected_indices.indices: ['.opendistro-alerting-config', '.opendistro-ism-*']
+
diff --git a/roles/odfees/templates/securityconfig/internal_users.yml.j2 b/roles/odfees/templates/securityconfig/internal_users.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..4d57f37b9690d9a51d8722484827a5d689a6a71a
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/internal_users.yml.j2
@@ -0,0 +1,56 @@
+---
+# This is the internal user database
+# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+
+_meta:
+ type: "internalusers"
+ config_version: 2
+
+# Define your internal users here
+
+## Demo users
+
+admin:
+ hash: "{{adminhashpwd}}"
+ reserved: true
+ backend_roles:
+ - "admin"
+ description: "Demo admin user"
+
+kibanaserver:
+ hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
+ reserved: true
+ description: "Demo kibanaserver user"
+
+kibanaro:
+ hash: "$2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC"
+ reserved: false
+ backend_roles:
+ - "kibanauser"
+ - "readall"
+ attributes:
+ attribute1: "value1"
+ attribute2: "value2"
+ attribute3: "value3"
+ description: "Demo kibanaro user"
+
+logstash:
+ hash: "$2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2"
+ reserved: false
+ backend_roles:
+ - "logstash"
+ description: "Demo logstash user"
+
+readall:
+ hash: "$2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2"
+ reserved: false
+ backend_roles:
+ - "readall"
+ description: "Demo readall user"
+
+snapshotrestore:
+ hash: "$2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W"
+ reserved: false
+ backend_roles:
+ - "snapshotrestore"
+ description: "Demo snapshotrestore user"
diff --git a/roles/odfees/templates/securityconfig/roles.yml b/roles/odfees/templates/securityconfig/roles.yml
new file mode 100644
index 0000000000000000000000000000000000000000..30d2387209b060909456e355be0ce3f3146b61e8
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/roles.yml
@@ -0,0 +1,39 @@
+_meta:
+ type: "roles"
+ config_version: 2
+
+# Restrict users so they can only view visualization and dashboard on kibana
+kibana_read_only:
+ reserved: true
+
+# The security REST API access role is used to assign specific users access to change the security settings through the REST API.
+security_rest_api_access:
+ reserved: true
+
+# Allows users to view alerts
+alerting_view_alerts:
+ reserved: true
+ index_permissions:
+ - index_patterns:
+ - ".opendistro-alerting-alert*"
+ allowed_actions:
+ - read
+
+# Allows users to view and acknowledge alerts
+alerting_crud_alerts:
+ reserved: true
+ index_permissions:
+ - index_patterns:
+ - ".opendistro-alerting-alert*"
+ allowed_actions:
+ - crud
+
+# Allows users to use all alerting functionality
+alerting_full_access:
+ reserved: true
+ index_permissions:
+ - index_patterns:
+ - ".opendistro-alerting-config"
+ - ".opendistro-alerting-alert*"
+ allowed_actions:
+ - crud
diff --git a/roles/odfees/templates/securityconfig/roles_mapping.yml b/roles/odfees/templates/securityconfig/roles_mapping.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4ebe922b2dfd9981022d407c2ddd8a9470a78f7b
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/roles_mapping.yml
@@ -0,0 +1,51 @@
+---
+# In this file users, backendroles and hosts can be mapped to Open Distro Security roles.
+# Permissions for Opendistro roles are configured in roles.yml
+
+_meta:
+ type: "rolesmapping"
+ config_version: 2
+
+# Define your roles mapping here
+
+## Demo roles mapping
+
+all_access:
+ reserved: false
+ backend_roles:
+ - "admin"
+ users:
+ - "bozidar.proevski"
+ description: "Maps admin to all_access"
+
+own_index:
+ reserved: false
+ users:
+ - "*"
+ description: "Allow full access to an index named like the username"
+
+logstash:
+ reserved: false
+ backend_roles:
+ - "logstash"
+
+kibana_user:
+ reserved: false
+ backend_roles:
+ - "kibanauser"
+ description: "Maps kibanauser to kibana_user"
+
+readall:
+ reserved: false
+ backend_roles:
+ - "readall"
+
+manage_snapshots:
+ reserved: false
+ backend_roles:
+ - "snapshotrestore"
+
+kibana_server:
+ reserved: true
+ users:
+ - "kibanaserver"
diff --git a/roles/odfees/templates/securityconfig/tenants.yml b/roles/odfees/templates/securityconfig/tenants.yml
new file mode 100644
index 0000000000000000000000000000000000000000..04104dce00de58fa5b08c4724299dba20c16580e
--- /dev/null
+++ b/roles/odfees/templates/securityconfig/tenants.yml
@@ -0,0 +1,11 @@
+---
+_meta:
+ type: "tenants"
+ config_version: 2
+
+# Define your tenants here
+
+## Demo tenants
+admin_tenant:
+ reserved: false
+ description: "Demo tenant for admin user"
diff --git a/roles/odfees/templates/sysconfig_elasticsearch.j2 b/roles/odfees/templates/sysconfig_elasticsearch.j2
new file mode 100644
index 0000000000000000000000000000000000000000..60b69e2bcf1ca61478e7b94015344c15455ebc47
--- /dev/null
+++ b/roles/odfees/templates/sysconfig_elasticsearch.j2
@@ -0,0 +1,51 @@
+################################
+# Elasticsearch
+################################
+
+# Elasticsearch home directory
+ES_HOME=/usr/share/elasticsearch
+
+# Elasticsearch Java path
+#JAVA_HOME=
+
+# Elasticsearch configuration directory
+ES_PATH_CONF=/usr/share/elasticsearch/config
+
+# Elasticsearch PID directory
+#PID_DIR=/var/run/elasticsearch
+
+# Additional Java OPTS
+#ES_JAVA_OPTS=
+
+# Configure restart on package upgrade (true, every other setting will lead to not restarting)
+#RESTART_ON_UPGRADE=true
+
+################################
+# Elasticsearch service
+################################
+
+# SysV init.d
+#
+# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
+ES_STARTUP_SLEEP_TIME=5
+
+################################
+# System properties
+################################
+
+# Specifies the maximum file descriptor number that can be opened by this process
+# When using Systemd, this setting is ignored and the LimitNOFILE defined in
+# /usr/lib/systemd/system/elasticsearch.service takes precedence
+#MAX_OPEN_FILES=65535
+
+# The maximum number of bytes of memory that may be locked into RAM
+# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
+# in elasticsearch.yml.
+# When using systemd, LimitMEMLOCK must be set in a unit file such as
+# /etc/systemd/system/elasticsearch.service.d/override.conf.
+#MAX_LOCKED_MEMORY=unlimited
+
+# Maximum number of VMA (Virtual Memory Areas) a process can own
+# When using Systemd, this setting is ignored and the 'vm.max_map_count'
+# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
+#MAX_MAP_COUNT=262144
diff --git a/roles/odfees/vars/main.yml b/roles/odfees/vars/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfekibana/defaults/main.yml b/roles/odfekibana/defaults/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfekibana/files/.empty b/roles/odfekibana/files/.empty
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfekibana/handlers/main.yml b/roles/odfekibana/handlers/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfekibana/meta/main.yml b/roles/odfekibana/meta/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/roles/odfekibana/tasks/main.yml b/roles/odfekibana/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..fa69837af75f25ec5a3956e70f6e72fe103482bb
--- /dev/null
+++ b/roles/odfekibana/tasks/main.yml
@@ -0,0 +1,101 @@
+---
+
+#- name: Create config directory
+# file:
+# name: config
+# state: directory
+# mode: 0700
+# tags:
+# - start
+
+- name: Copy certificates in odfe kibana conf dir
+ copy:
+ src: "{{ item }}"
+ dest: "config/{{ item }}"
+ mode: 0600
+ with_items:
+ - "{{ inventory_hostname }}.p12"
+ - "{{ inventory_hostname }}.crt"
+ - "{{ inventory_hostname }}.key"
+ - cacerts.jks
+# - "{{nifiadmin.0[0]}}.p12"
+ tags:
+ - start
+
+#- name: Configure sysconfig
+# template:
+# src: sysconfig_elasticsearch.j2
+# dest: sysconfig_elasticsearch
+# tags:
+# - start
+#
+#- name: Copy sysconfig to /etc
+# command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
+# tags:
+# - start
+
+# lineinfile:
+# path: /etc/sysconfig/elasticsearch
+# regexp: '^ES_PATH_CONF='
+# line: ES_PATH_CONF=/usr/share/elasticsearch/config
+# tags:
+# - start
+- name: Configure odfe kibana properties
+ template:
+ src: "{{item}}.j2"
+ dest: "config/{{item}}"
+ with_items:
+ - kibana.yml
+ tags:
+ - start
+
+- name: Configure odfe kibana start script
+ template:
+ src: "{{item}}.j2"
+ dest: "{{item}}"
+ mode: 0750
+ with_items:
+ - startkibana.sh
+ tags:
+ - start
+
+#- name: Exit here to test ODFE
+# meta: end_play
+# tags:
+# - start
+
+- name: Start OpenDistro Kibana for Elasticsearch
+ command: /usr/share/kibana/startkibana.sh
+ #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml &
+ #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &"
+ tags:
+ - start
+
+- name: Wait for Kibana
+ wait_for:
+ host: "{{groups['odfekibanacontainers'][0]}}"
+ port: 5601
+ state: started
+ delay: 5
+ tags:
+ - start
+
+#- name: check reachable hosts
+# gather_facts: no
+# tasks:
+# - command: ping -c1 {{ inventory_hostname }}
+# delegate_to: localhost
+# register: ping_result
+# ignore_errors: yes
+# - group_by: key=reachable
+# when: ping_result|success
+
+#- name: Stop OpenDistro Kibana for Elasticsearch
+# command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+# tags:
+# - stop
+
+- name: Stop OpenDistro Kibana for Elasticsearch
+ command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+ tags:
+ - stop
diff --git a/roles/odfekibana/templates/kibana.yml.j2 b/roles/odfekibana/templates/kibana.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..c1242977e01b7621cb52848505f7a30f65687f68
--- /dev/null
+++ b/roles/odfekibana/templates/kibana.yml.j2
@@ -0,0 +1,65 @@
+---
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# Description:
+# Default Kibana configuration from kibana-docker.
+
+#logging.verbose: true
+cpu.cgroup.path.override: /
+cpuacct.cgroup.path.override: /
+pid.file: {{inventory_hostname}}.pid
+
+server.name: {{inventory_hostname}}
+server.host: "{{inventory_hostname}}"
+#elasticsearch.hosts: https://localhost:9200
+elasticsearch.hosts: https://{{groups['odfeescontainers'][0]}}:9200
+elasticsearch.ssl.verificationMode: none
+elasticsearch.username: kibanaserver
+elasticsearch.password: kibanaserver
+elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"]
+
+opendistro_security.multitenancy.enabled: true
+opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
+opendistro_security.readonly_mode.roles: ["kibana_read_only"]
+
+newsfeed.enabled: false
+telemetry.optIn: false
+telemetry.enabled: false
+
+#opendistro_security.auth.type: "openid"
+#opendistro_security.openid.connect_url: "https://dsldev.gn4-3-wp8-soc.sunet.se:10443/auth/realms/GN43WP8T31SOC1/.well-known/openid-configuration"
+#opendistro_security.openid.client_id: "dsoclab-kibana"
+#opendistro_security.openid.client_secret: "ccaa137f-2a2b-48ae-bcce-9e1fbcbbf181"
+#opendistro_security.openid.root_ca: /usr/share/kibana/config/dslca.crt
+## opendistro_security.openid.root_ca: /usr/share/kibana/config/gn43wp8t31ca.crt
+#opendistro_security.openid.base_redirect_url: "https://dsldev.gn4-3-wp8-soc.sunet.se:5601"
+
+opendistro_security.cookie.secure: true
+opendistro_security.cookie.password: "MezgW6l2v9BWi6wEwbEn4gaqJZbWGPSI"
+
+server.ssl.enabled: true
+#server.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
+#server.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
+server.ssl.keystore.path: /usr/share/kibana/config/{{inventory_hostname}}.p12
+server.ssl.keystore.password: {{kspass}}
+#server.ssl.certificateAuthorities:
+#server.ssl.truststore.path: jks (p12?)
+#server.ssl.truststore.password:
+
+
+#elasticsearch.ssl.certificate: /usr/share/kibana/config/odfe-kibana.crt
+#elasticsearch.ssl.key: /usr/share/kibana/config/odfe-kibana.key
+#elasticsearch.ssl.certificateAuthorities: /usr/share/kibana/config/dslca.crt
+
+opendistro_security.allow_client_certificates: true
diff --git a/roles/odfekibana/templates/startkibana.sh.j2 b/roles/odfekibana/templates/startkibana.sh.j2
new file mode 100644
index 0000000000000000000000000000000000000000..74039208775785dc27dd1349cf2debc0889a9dc5
--- /dev/null
+++ b/roles/odfekibana/templates/startkibana.sh.j2
@@ -0,0 +1,6 @@
+#!/bin/bash -x
+#exec /usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml --verbose > kblog 2>&1 &
+/usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml > kblog 2>&1 &
+# disown
+
+
diff --git a/roles/odfekibana/vars/main.yml b/roles/odfekibana/vars/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/soctools-inventory b/soctools-inventory
index 5f453c399ec17c7d7453350ab3fdf96d1e5f4ba2..dfec58ef286f8d823f1e3388452ef3ff0dc0b7cc 100644
--- a/soctools-inventory
+++ b/soctools-inventory
@@ -6,6 +6,13 @@ dsoclab-nifi-1 ansible_connection=docker
dsoclab-nifi-2 ansible_connection=docker
dsoclab-nifi-3 ansible_connection=docker
+[odfeescontainers]
+dsoclab-odfe-1 ansible_connection=docker
+dsoclab-odfe-2 ansible_connection=docker
+
+[odfekibanacontainers]
+dsoclab-kibana ansible_connection=docker
+
[nginx]
dsoclab-nginx ansible_connection=docker
diff --git a/startsoctools.yml b/startsoctools.yml
index 216a9c9c1ca2416a99f68ebc49b021fe786a2a23..aa8cc1108361c4204e18bbb3928598180a29cb9c 100644
--- a/startsoctools.yml
+++ b/startsoctools.yml
@@ -15,3 +15,12 @@
roles:
- nifi
+- name: Reconfigure and start OpenDistro for Elasticsearch
+ hosts: odfeescontainers
+ roles:
+ - odfees
+
+- name: Reconfigure and start OpenDistro Kibana for Elasticsearch
+ hosts: odfekibanacontainers
+ roles:
+ - odfekibana