Added new roles

role  odfees (Open Distro for Elasticsearch) and
role odfekibana (Open Distro for Elasticsearch Kibana)

roles/odfees/tasks/main.yml

+---
+
+- name: Create config directory
+  file:
+    name: config
+    state: directory
+    mode: 0700
+  tags:
+    - start
+
+- name: Copy certificates in odfe conf dir
+  copy:
+    src:  "{{ item }}"
+    dest: "config/{{ item }}"
+    mode: 0600
+  with_items:
+    - "{{ inventory_hostname }}.p12"
+    - cacerts.jks
+    - "{{nifiadmin.0[0]}}.p12"
+  tags:
+    - start
+
+- name: Configure sysconfig
+  template:
+    src: sysconfig_elasticsearch.j2
+    dest: sysconfig_elasticsearch
+  tags:
+    - start
+
+- name: Copy sysconfig to /etc
+  command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
+  tags: 
+    - start
+
+#  lineinfile:
+#    path: /etc/sysconfig/elasticsearch
+#    regexp: '^ES_PATH_CONF='
+#    line: ES_PATH_CONF=/usr/share/elasticsearch/config
+#  tags:
+#    - start
+- name: Configure odfe properties
+  template:
+    src: "config/{{item}}.j2"
+    dest: "config/{{item}}"
+  with_items:
+    - elasticsearch.yml
+    - jvm.options
+    - log4j2.properties
+  tags:
+    - start
+
+- name: Change password for admin
+  command: "bash plugins/opendistro_security/tools/hash.sh -p {{odfees_adminpass}}"
+  register: adminhash
+  # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
+  tags:
+    - start
+
+- set_fact:
+    adminhashpwd: "{{ adminhash.stdout }}"
+    #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}"
+  tags:
+    - start
+
+- name: Configure opendistro_security properties
+  template:
+    src: "securityconfig/{{item}}.j2"
+    dest: "plugins/opendistro_security/securityconfig/{{item}}"
+    #dest: "{{item}}"
+  with_items:
+    - internal_users.yml
+    - config.yml
+  tags:
+    - start
+
+#- name: Exit here to test ODFE
+#  meta: end_play
+#  tags:
+#    - start
+
+- name: Start OpenDistro for Elasticsearch
+  command: "/usr/share/elasticsearch/bin/elasticsearch -p {{ inventory_hostname }}.pid -d"
+  tags:
+    - start
+
+- name: Wait for ElasticSearch
+  wait_for:
+    host: "{{groups['odfeescontainers'][0]}}"
+    port: 9200
+    state: started
+    delay: 5
+  tags:
+    - start
+
+- name: Configure OpenDistro security
+  command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{nifiadmin.0[0]}}.p12' -kspass {{nifiadmin.0[1]}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{tspass}} -cn dsoclab-cluster"
+  when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
+  tags:
+    - start
+
+#- name: check reachable hosts
+#  gather_facts: no
+#  tasks:
+#    - command: ping -c1 {{ inventory_hostname }}
+#      delegate_to: localhost
+#      register: ping_result
+#      ignore_errors: yes
+#    - group_by: key=reachable
+#      when: ping_result|success
+
+- name: Stop OpenDistro for Elasticsearch
+  command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+  tags:
+    - stop
+

roles/odfees/templates/config/elasticsearch.yml.j2

+cluster.name: "dsoclab-cluster"
+#network.host: 0.0.0.0
+network.host: {{ inventory_hostname }}
+discovery.seed_hosts:
+{% for odfees in groups['odfeescontainers'] %}
+  - {{ odfees }}
+{% endfor %}
+#discovery.type: single-node
+transport.port: 9300
+
+path.logs: /usr/share/elasticsearch/logs
+# # minimum_master_nodes need to be explicitly set when bound on a public IP
+# # set to 1 to allow single node clusters
+# # Details: https://github.com/elastic/elasticsearch/pull/17288
+#discovery.zen.minimum_master_nodes: 1
+
+# # Breaking change in 7.0
+# # https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#breaking_70_discovery_changes
+cluster.initial_master_nodes: 
+{% for odfees in groups['odfeescontainers'] %}
+  - {{ odfees }}
+{% endfor %}
+
+#    - elasticsearch1
+#    - docker-test-node-1 
+######## Start OpenDistro for Elasticsearch Security Demo Configuration ########
+# WARNING: revise all the lines below before you go into production
+# opendistro_security.ssl.transport.pemcert_filepath: esnode.pem
+# opendistro_security.ssl.transport.pemkey_filepath: esnode-key.pem
+
+opendistro_security.ssl.transport.keystore_type: pkcs12
+opendistro_security.ssl.transport.keystore_filepath: {{ inventory_hostname }}.p12
+opendistro_security.ssl.transport.keystore_password: {{ kspass }}
+#opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
+opendistro_security.ssl.transport.truststore_type: jks
+opendistro_security.ssl.transport.truststore_filepath: cacerts.jks
+opendistro_security.ssl.transport.truststore_password: {{ tspass }}
+opendistro_security.ssl.transport.enforce_hostname_verification: false
+
+opendistro_security.ssl.http.enabled: true
+# opendistro_security.ssl.http.pemcert_filepath: esnode.pem
+# opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem
+opendistro_security.ssl.http.keystore_type: pkcs12
+opendistro_security.ssl.http.keystore_filepath: {{ inventory_hostname }}.p12
+opendistro_security.ssl.http.keystore_password: {{ kspass }}
+opendistro_security.ssl.http.truststore_type: jks
+opendistro_security.ssl.http.truststore_filepath: cacerts.jks
+opendistro_security.ssl.http.truststore_password: {{ tspass }}
+#opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem
+#opendistro_security.ssl.http.clientauth_mode: optional
+opendistro_security.allow_unsafe_democertificates: false
+opendistro_security.allow_default_init_securityindex: false
+opendistro_security.authcz.admin_dn:
+  - "CN={{ nifiadmin[0][0] }}"
+
+opendistro_security.nodes_dn:
+{% for odfees in groups['odfeescontainers'] %}
+  - "CN={{ odfees }}"
+{% endfor %}
+
+opendistro_security.audit.type: internal_elasticsearch
+opendistro_security.enable_snapshot_restore_privilege: true
+opendistro_security.check_snapshot_restore_write_privileges: true
+opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
+cluster.routing.allocation.disk.threshold_enabled: false
+node.max_local_storage_nodes: 3
+######## End OpenDistro for Elasticsearch Security Demo Configuration ########

roles/odfees/templates/config/jvm.options.j2

+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms512m
+-Xmx512m
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1GC is only supported on JDK version 10 or later.
+# To use G1GC uncomment the lines below.
+# 10-:-XX:-UseConcMarkSweepGC
+# 10-:-XX:-UseCMSInitiatingOccupancyOnly
+# 10-:-XX:+UseG1GC
+# 10-:-XX:InitiatingHeapOccupancyPercent=75
+
+## DNS cache policy
+# cache ttl in seconds for positive DNS lookups noting that this overrides the
+# JDK security property networkaddress.cache.ttl; set to -1 to cache forever
+-Des.networkaddress.cache.ttl=60
+# cache ttl in seconds for negative DNS lookups noting that this overrides the
+# JDK security property networkaddress.cache.negative ttl; set to -1 to cache
+# forever
+-Des.networkaddress.cache.negative.ttl=10
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# explicitly set the stack size
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# turn off a JDK optimization that throws away stack traces for common
+# exceptions because stack traces are important for debugging
+-XX:-OmitStackTraceInFastThrow
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=data
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=logs/hs_err_pid%p.log
+
+## JDK 8 GC logging
+
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:logs/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
+# time/date parsing will break in an incompatible way for some date patterns and locals
+9-:-Djava.locale.providers=COMPAT

roles/odfees/templates/config/log4j2.properties.j2

+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console

roles/odfees/templates/securityconfig/action_groups.yml

+_meta:
+  type: "actiongroups"
+  config_version: 2
\ No newline at end of file

roles/odfees/templates/securityconfig/config.yml.j2

+---
+
+# This is the main Open Distro Security configuration file where authentication
+# and authorization is defined.
+#
+# You need to configure at least one authentication domain in the authc of this file.
+# An authentication domain is responsible for extracting the user credentials from
+# the request and for validating them against an authentication backend like Active Directory for example.
+#
+# If more than one authentication domain is configured the first one which succeeds wins.
+# If all authentication domains fail then the request is unauthenticated.
+# In this case an exception is thrown and/or the HTTP status is set to 401.
+#
+# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect
+# the roles from a given backend for the authenticated user.
+#
+# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both.
+#        http_enabled: true
+#        transport_enabled: true
+#
+# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to
+# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated.
+# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous"
+# and one role named "anonymous_backendrole".
+# If you enable anonymous authentication all HTTP authenticators will not challenge.
+#
+#
+# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert"
+# first and the challenging one last.
+# Because it's not possible to challenge a client with two different authentication methods (for example
+# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation
+# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request.
+#
+# Default value of the challenge flag is true.
+#
+#
+# HTTP
+#   basic (challenging)
+#   proxy (not challenging, needs xff)
+#   kerberos (challenging)
+#   clientcert (not challenging, needs https)
+#   jwt (not challenging)
+#   host (not challenging) #DEPRECATED, will be removed in a future version.
+#                          host based authentication is configurable in roles_mapping
+
+# Authc
+#   internal
+#   noop
+#   ldap
+
+# Authz
+#   ldap
+#   noop
+
+
+
+_meta:
+  type: "config"
+  config_version: 2
+
+config:
+  dynamic:
+    # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index
+    # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default)
+    # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently
+    #filtered_alias_mode: warn
+    #do_not_fail_on_forbidden: false
+    #kibana:
+    # Kibana multitenancy
+    #multitenancy_enabled: true
+    #server_username: kibanaserver
+    #index: '.kibana'
+    http:
+      anonymous_auth_enabled: false
+      xff:
+        enabled: false
+        internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern
+        #internalProxies: '.*' # trust all internal proxies, regex pattern
+        #remoteIpHeader:  'x-forwarded-for'
+        ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help
+        ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For
+        ###### and here https://tools.ietf.org/html/rfc7239
+        ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve
+    authc:
+      kerberos_auth_domain:
+        http_enabled: false
+        transport_enabled: false
+        order: 6
+        http_authenticator:
+          type: kerberos
+          challenge: true
+          config:
+            # If true a lot of kerberos/security related debugging output will be logged to standard out
+            krb_debug: false
+            # If true then the realm will be stripped from the user name
+            strip_realm_from_principal: true
+        authentication_backend:
+          type: noop
+      basic_internal_auth_domain:
+        description: "Authenticate via HTTP Basic against internal users database"
+        http_enabled: true
+        transport_enabled: true
+        order: 4
+        http_authenticator:
+          type: basic
+          challenge: true
+        authentication_backend:
+          type: intern
+      openid_auth_domain:
+        http_enabled: true
+        transport_enabled: true
+        order: 1
+        http_authenticator:
+          type: openid
+          challenge: false
+          config:
+            subject_key: {{openid_subjkey}} 
+            roles_key: roles
+            openid_connect_url: https://{{dslproxy}}:10443/auth/realms/{{openid_realm}}/.well-known/openid-configuration
+            enable_ssl: true
+            verify_hostnames: false
+            # pemtrustedcas_filepath: /usr/share/elasticsearch/config/dslca.crt
+        authentication_backend:
+          type: noop
+      proxy_auth_domain:
+        description: "Authenticate via proxy"
+        http_enabled: false
+        transport_enabled: false
+        order: 3
+        http_authenticator:
+          type: proxy
+          challenge: false
+          config:
+            user_header: "x-proxy-user"
+            roles_header: "x-proxy-roles"
+        authentication_backend:
+          type: noop
+      jwt_auth_domain:
+        description: "Authenticate via Json Web Token"
+        http_enabled: false
+        transport_enabled: false
+        order: 0
+        http_authenticator:
+          type: jwt
+          challenge: false
+          config:
+            signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key"
+            jwt_header: "Authorization"
+            jwt_url_parameter: null
+            roles_key: null
+            subject_key: null
+        authentication_backend:
+          type: noop
+      clientcert_auth_domain:
+        description: "Authenticate via SSL client certificates"
+        http_enabled: false
+        transport_enabled: false
+        order: 2
+        http_authenticator:
+          type: clientcert
+          config:
+            username_attribute: cn #optional, if omitted DN becomes username
+          challenge: false
+        authentication_backend:
+          type: noop
+      ldap:
+        description: "Authenticate via LDAP or Active Directory"
+        http_enabled: false
+        transport_enabled: false
+        order: 5
+        http_authenticator:
+          type: basic
+          challenge: false
+        authentication_backend:
+          # LDAP authentication backend (authenticate users against a LDAP or Active Directory)
+          type: ldap
+          config:
+            # enable ldaps
+            enable_ssl: false
+            # enable start tls, enable_ssl should be false
+            enable_start_tls: false
+            # send client certificate
+            enable_ssl_client_auth: false
+            # verify ldap hostname
+            verify_hostnames: true
+            hosts:
+            - localhost:8389
+            bind_dn: null
+            password: null
+            userbase: 'ou=people,dc=example,dc=com'
+            # Filter to search for users (currently in the whole subtree beneath userbase)
+            # {0} is substituted with the username
+            usersearch: '(sAMAccountName={0})'
+            # Use this attribute from the user as username (if not set then DN is used)
+            username_attribute: null
+    authz:
+      roles_from_myldap:
+        description: "Authorize via LDAP or Active Directory"
+        http_enabled: false
+        transport_enabled: false
+        authorization_backend:
+          # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too)
+          type: ldap
+          config:
+            # enable ldaps
+            enable_ssl: false
+            # enable start tls, enable_ssl should be false
+            enable_start_tls: false
+            # send client certificate
+            enable_ssl_client_auth: false
+            # verify ldap hostname
+            verify_hostnames: true
+            hosts:
+            - localhost:8389
+            bind_dn: null
+            password: null
+            rolebase: 'ou=groups,dc=example,dc=com'
+            # Filter to search for roles (currently in the whole subtree beneath rolebase)
+            # {0} is substituted with the DN of the user
+            # {1} is substituted with the username
+            # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute
+            rolesearch: '(member={0})'
+            # Specify the name of the attribute which value should be substituted with {2} above
+            userroleattribute: null
+            # Roles as an attribute of the user entry
+            userrolename: disabled
+            #userrolename: memberOf
+            # The attribute in a role entry containing the name of that role, Default is "name".
+            # Can also be "dn" to use the full DN as rolename.
+            rolename: cn
+            # Resolve nested roles transitive (roles which are members of other roles and so on ...)
+            resolve_nested_roles: true
+            userbase: 'ou=people,dc=example,dc=com'
+            # Filter to search for users (currently in the whole subtree beneath userbase)
+            # {0} is substituted with the username
+            usersearch: '(uid={0})'
+            # Skip users matching a user name, a wildcard or a regex pattern
+            #skip_users:
+            #  - 'cn=Michael Jackson,ou*people,o=TEST'
+            #  - '/\S*/'
+      roles_from_another_ldap:
+        description: "Authorize via another Active Directory"
+        http_enabled: false
+        transport_enabled: false
+        authorization_backend:
+          type: ldap
+          #config goes here ...
+  #    auth_failure_listeners:
+  #      ip_rate_limiting:
+  #        type: ip
+  #        allowed_tries: 10
+  #        time_window_seconds: 3600
+  #        block_expiry_seconds: 600
+  #        max_blocked_clients: 100000
+  #        max_tracked_clients: 100000
+  #      internal_authentication_backend_limiting:
+  #        type: username
+  #        authentication_backend: intern
+  #        allowed_tries: 10
+  #        time_window_seconds: 3600
+  #        block_expiry_seconds: 600
+  #        max_blocked_clients: 100000
+  #        max_tracked_clients: 100000

roles/odfees/templates/securityconfig/elasticsearch.yml.example

+############## Open Distro Security configuration ###############
+
+###########################################################
+# Add the following settings to your standard elasticsearch.yml 
+# alongside with the Open Distro Security TLS settings.
+# Settings must always be the same on all nodes in the cluster.
+
+############## Common configuration settings ##############
+
+# Enable or disable the Open Distro Security advanced modules
+# By default advanced modules are enabled, you can switch
+# all advanced features off by setting the following key to false
+opendistro_security.advanced_modules_enabled: true
+
+# Specify a list of DNs which denote the other nodes in the cluster.
+# This settings support wildcards and regular expressions
+# This setting only has effect if 'opendistro_security.cert.intercluster_request_evaluator_class' is not set.
+opendistro_security.nodes_dn:
+  - "CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE"
+  - "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE"
+
+# Defines the DNs (distinguished names) of certificates
+# to which admin privileges should be assigned (mandatory)
+opendistro_security.authcz.admin_dn:
+  - "CN=kirk,OU=client,O=client,l=tEst, C=De"
+
+# Define how backend roles should be mapped to Open Distro Security roles
+# MAPPING_ONLY - mappings must be configured explicitely in roles_mapping.yml (default)
+# BACKENDROLES_ONLY - backend roles are mapped to Open Distro Security rules directly. Settings in roles_mapping.yml have no effect.
+# BOTH - backend roles are mapped to Open Distro Security roles mapped directly and via roles_mapping.yml in addition
+opendistro_security.roles_mapping_resolution: MAPPING_ONLY
+
+############## REST Management API configuration settings ##############
+# Enable or disable role based access to the REST management API
+# Default is that no role is allowed to access the REST management API.
+#opendistro_security.restapi.roles_enabled: ["all_access","xyz_role"]
+
+# Disable particular endpoints and their HTTP methods for roles. 
+# By default all endpoints/methods are allowed.
+#opendistro_security.restapi.endpoints_disabled.<role>.<endpoint>: <array of http methods>
+# Example:
+#opendistro_security.restapi.endpoints_disabled.all_access.ACTIONGROUPS: ["PUT","POST","DELETE"]
+#opendistro_security.restapi.endpoints_disabled.xyz_role.LICENSE: ["DELETE"]
+
+# The following endpoints exist:
+# ACTIONGROUPS
+# CACHE
+# CONFIG
+# ROLES
+# ROLESMAPPING
+# INTERNALUSERS
+# SYSTEMINFO
+# PERMISSIONSINFO
+
+############## Auditlog configuration settings ##############
+# General settings
+
+# Enable/disable rest request logging (default: true)
+#opendistro_security.audit.enable_rest: true
+# Enable/disable transport request logging (default: false)
+#opendistro_security.audit.enable_transport: false
+# Enable/disable bulk request logging (default: false)
+# If enabled all subrequests in bulk requests will be logged too
+#opendistro_security.audit.resolve_bulk_requests: false
+# Disable some categories
+#opendistro_security.audit.config.disabled_categories: ["AUTHENTICATED","GRANTED_PRIVILEGES"]
+# Disable some requests (wildcard or regex of actions or rest request paths)
+#opendistro_security.audit.ignore_requests: ["indices:data/read/*","*_bulk"]
+# Tune threadpool size, default is 10 and 0 means disabled
+#opendistro_security.audit.threadpool.size: 0
+# Tune threadpool max size queue length, default is 100000
+#opendistro_security.audit.threadpool.max_queue_len: 100000
+
+# If enable_request_details is true then the audit log event will also contain
+# details like the search query. Default is false. 
+#opendistro_security.audit.enable_request_details: true
+# Ignore users, e.g. do not log audit requests from that users (default: no ignored users)
+#opendistro_security.audit.ignore_users: ['kibanaserver','some*user','/also.*regex possible/']"
+
+# Destination of the auditlog events
+opendistro_security.audit.type: internal_elasticsearch
+#opendistro_security.audit.type: external_elasticsearch
+#opendistro_security.audit.type: debug
+#opendistro_security.audit.type: webhook
+
+# external_elasticsearch settings
+#opendistro_security.audit.config.http_endpoints: ['localhost:9200','localhost:9201','localhost:9202']"
+# Auditlog index can be a static one or one with a date pattern (default is 'auditlog6')
+#opendistro_security.audit.config.index: auditlog6 # make sure you secure this index properly
+#opendistro_security.audit.config.index: "'auditlog6-'YYYY.MM.dd" #rotates index daily - make sure you secure this index properly
+#opendistro_security.audit.config.type: auditlog
+#opendistro_security.audit.config.username: auditloguser
+#opendistro_security.audit.config.password: auditlogpassword
+#opendistro_security.audit.config.enable_ssl: false
+#opendistro_security.audit.config.verify_hostnames: false
+#opendistro_security.audit.config.enable_ssl_client_auth: false
+#opendistro_security.audit.config.cert_alias: mycert
+#opendistro_security.audit.config.pemkey_filepath: key.pem
+#opendistro_security.audit.config.pemkey_content: <...pem base 64 content>
+#opendistro_security.audit.config.pemkey_password: secret
+#opendistro_security.audit.config.pemcert_filepath: cert.pem
+#opendistro_security.audit.config.pemcert_content: <...pem base 64 content>
+#opendistro_security.audit.config.pemtrustedcas_filepath: ca.pem
+#opendistro_security.audit.config.pemtrustedcas_content: <...pem base 64 content>
+
+# webhook settings
+#opendistro_security.audit.config.webhook.url: "http://mywebhook/endpoint"
+# One of URL_PARAMETER_GET,URL_PARAMETER_POST,TEXT,JSON,SLACK
+#opendistro_security.audit.config.webhook.format: JSON
+#opendistro_security.audit.config.webhook.ssl.verify: false
+#opendistro_security.audit.config.webhook.ssl.pemtrustedcas_filepath: ca.pem
+#opendistro_security.audit.config.webhook.ssl.pemtrustedcas_content: <...pem base 64 content>
+
+# log4j settings
+#opendistro_security.audit.config.log4j.logger_name: auditlogger
+#opendistro_security.audit.config.log4j.level: INFO
+
+############## Kerberos configuration settings ##############
+# If Kerberos authentication should be used you have to configure:
+
+# The Path to the krb5.conf file
+# Can be absolute or relative to the Elasticsearch config directory
+#opendistro_security.kerberos.krb5_filepath: '/etc/krb5.conf'
+            
+# The Path to the keytab where the acceptor_principal credentials are stored.           
+# Must be relative to the Elasticsearch config directory
+#opendistro_security.kerberos.acceptor_keytab_filepath: 'eskeytab.tab'
+
+# Acceptor (Server) Principal name, must be present in acceptor_keytab_path file
+#opendistro_security.kerberos.acceptor_principal: 'HTTP/localhost'
+
+############## Advanced configuration settings ##############
+# Enable transport layer impersonation
+# Allow DNs (distinguished names) to impersonate as other users
+#opendistro_security.authcz.impersonation_dn:
+#  "CN=spock,OU=client,O=client,L=Test,C=DE":
+#    - worf
+#  "cn=webuser,ou=IT,ou=IT,dc=company,dc=com":
+#    - user2
+#    - user1
+
+# Enable rest layer impersonation
+# Allow users to impersonate as other users
+#opendistro_security.authcz.rest_impersonation_user:
+#  "picard":
+#    - worf
+#  "john":
+#    - steve
+#    - martin
+
+# If this is set to true Open Distro Security will automatically initialize the configuration index
+# with the files in the config directory if the index does not exist.
+# WARNING: This will use well-known default passwords.
+#          Use only in a private network/environment.
+#opendistro_security.allow_default_init_securityindex: false
+
+# If this is set to true then allow to startup with demo certificates.
+# These are certificates issued by floragunn GmbH for demo purposes.
+# WARNING: This certificates are well known and therefore unsafe
+#          Use only in a private network/environment.
+#opendistro_security.allow_unsafe_democertificates: false
+
+############## Expert settings ##############
+# WARNING: Expert settings, do only use if you know what you are doing
+# If you set wrong values here this this could be a security risk
+# or make Open Distro Security stop working
+
+# Name of the index where .opendistro_security stores its configuration.
+
+#opendistro_security.config_index_name: .opendistro_security
+
+# This defines the OID of server node certificates
+#opendistro_security.cert.oid: '1.2.3.4.5.5'
+
+# This specifies the implementation of com.amazon.opendistroforelasticsearch.security.transport.InterClusterRequestEvaluator
+# that is used to determine inter-cluster request.
+# Instances of com.amazon.opendistroforelasticsearch.security.transport.InterClusterRequestEvaluator must implement a single argument
+# constructor that takes an org.elasticsearch.common.settings.Settings
+#opendistro_security.cert.intercluster_request_evaluator_class: com.amazon.opendistroforelasticsearch.security.transport.DefaultInterClusterRequestEvaluator
+
+# Allow snapshot restore for normal users
+# By default only requests signed by an admin TLS certificate can do this
+# To enable snapshot restore for normal users set 'opendistro_security.enable_snapshot_restore_privilege: true'
+# The user who wants to restore a snapshot must have the 'cluster:admin/snapshot/restore' privilege and must also have
+# "indices:admin/create" and "indices:data/write/index" for the indices to be restores.
+# A snapshot can only be restored when it does not contain global state and does not restore the '.opendistro_security' index
+# If 'opendistro_security.check_snapshot_restore_write_privileges: false' is set then the additional indices checks are omitted.
+
+# This makes it less secure.
+#opendistro_security.enable_snapshot_restore_privilege: true
+#opendistro_security.check_snapshot_restore_write_privileges: false
+
+# Authentication cache timeout in minutes (A value of 0 disables caching, default is 60)
+#opendistro_security.cache.ttl_minutes: 60
+
+# Disable Open Distro Security
+# WARNING: This can expose your configuration (including passwords) to the public.
+#opendistro_security.disabled: false
+
+
+# Protected indices are even more secure than normal indices. These indices require a role to access like any other index, but they require an additional role
+# to be visible, listed in the opendistro_security.protected_indices.roles setting.
+# Enable protected indices
+# opendistro_security.protected_indices.enabled: true
+# Specify a list of roles a user must be member of to touch any protected index.
+# opendistro_security.protected_indices.roles: ['all_access']
+# Specify a list of indices to mark as protected. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role.
+# opendistro_security.protected_indices.indices: ['.opendistro-alerting-config', '.opendistro-ism-*']
+

roles/odfees/templates/securityconfig/internal_users.yml.j2

+---
+# This is the internal user database
+# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+
+_meta:
+  type: "internalusers"
+  config_version: 2
+
+# Define your internal users here
+
+## Demo users
+
+admin:
+  hash: "{{adminhashpwd}}"
+  reserved: true
+  backend_roles:
+  - "admin"
+  description: "Demo admin user"
+
+kibanaserver:
+  hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H."
+  reserved: true
+  description: "Demo kibanaserver user"
+
+kibanaro:
+  hash: "$2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC"
+  reserved: false
+  backend_roles:
+  - "kibanauser"
+  - "readall"
+  attributes:
+    attribute1: "value1"
+    attribute2: "value2"
+    attribute3: "value3"
+  description: "Demo kibanaro user"
+
+logstash:
+  hash: "$2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2"
+  reserved: false
+  backend_roles:
+  - "logstash"
+  description: "Demo logstash user"
+
+readall:
+  hash: "$2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2"
+  reserved: false
+  backend_roles:
+  - "readall"
+  description: "Demo readall user"
+
+snapshotrestore:
+  hash: "$2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W"
+  reserved: false
+  backend_roles:
+  - "snapshotrestore"
+  description: "Demo snapshotrestore user"

roles/odfees/templates/securityconfig/roles.yml

+_meta:
+  type: "roles"
+  config_version: 2
+
+# Restrict users so they can only view visualization and dashboard on kibana
+kibana_read_only:
+  reserved: true
+
+# The security REST API access role is used to assign specific users access to change the security settings through the REST API.
+security_rest_api_access:
+  reserved: true
+ 
+# Allows users to view alerts
+alerting_view_alerts:
+  reserved: true
+  index_permissions:
+    - index_patterns:
+      - ".opendistro-alerting-alert*"
+      allowed_actions:
+        - read 
+
+# Allows users to view and acknowledge alerts
+alerting_crud_alerts:
+  reserved: true
+  index_permissions:
+    - index_patterns:
+      - ".opendistro-alerting-alert*"
+      allowed_actions:
+       - crud 
+
+# Allows users to use all alerting functionality
+alerting_full_access:
+  reserved: true
+  index_permissions:
+    - index_patterns:
+      - ".opendistro-alerting-config"
+      - ".opendistro-alerting-alert*"
+      allowed_actions:
+        - crud 

roles/odfees/templates/securityconfig/roles_mapping.yml

+---
+# In this file users, backendroles and hosts can be mapped to Open Distro Security roles.
+# Permissions for Opendistro roles are configured in roles.yml
+
+_meta:
+  type: "rolesmapping"
+  config_version: 2
+
+# Define your roles mapping here
+
+## Demo roles mapping
+
+all_access:
+  reserved: false
+  backend_roles:
+  - "admin"
+  users:
+  - "bozidar.proevski"
+  description: "Maps admin to all_access"
+
+own_index:
+  reserved: false
+  users:
+  - "*"
+  description: "Allow full access to an index named like the username"
+
+logstash:
+  reserved: false
+  backend_roles:
+  - "logstash"
+
+kibana_user:
+  reserved: false
+  backend_roles:
+  - "kibanauser"
+  description: "Maps kibanauser to kibana_user"
+
+readall:
+  reserved: false
+  backend_roles:
+  - "readall"
+
+manage_snapshots:
+  reserved: false
+  backend_roles:
+  - "snapshotrestore"
+
+kibana_server:
+  reserved: true
+  users:
+  - "kibanaserver"

roles/odfees/templates/securityconfig/tenants.yml

+---
+_meta:
+  type: "tenants"
+  config_version: 2
+
+# Define your tenants here
+
+## Demo tenants
+admin_tenant:
+  reserved: false
+  description: "Demo tenant for admin user"

roles/odfees/templates/sysconfig_elasticsearch.j2

+################################
+# Elasticsearch
+################################
+
+# Elasticsearch home directory
+ES_HOME=/usr/share/elasticsearch
+
+# Elasticsearch Java path
+#JAVA_HOME=
+
+# Elasticsearch configuration directory
+ES_PATH_CONF=/usr/share/elasticsearch/config
+
+# Elasticsearch PID directory
+#PID_DIR=/var/run/elasticsearch
+
+# Additional Java OPTS
+#ES_JAVA_OPTS=
+
+# Configure restart on package upgrade (true, every other setting will lead to not restarting)
+#RESTART_ON_UPGRADE=true
+
+################################
+# Elasticsearch service
+################################
+
+# SysV init.d
+#
+# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
+ES_STARTUP_SLEEP_TIME=5
+
+################################
+# System properties
+################################
+
+# Specifies the maximum file descriptor number that can be opened by this process
+# When using Systemd, this setting is ignored and the LimitNOFILE defined in
+# /usr/lib/systemd/system/elasticsearch.service takes precedence
+#MAX_OPEN_FILES=65535
+
+# The maximum number of bytes of memory that may be locked into RAM
+# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
+# in elasticsearch.yml.
+# When using systemd, LimitMEMLOCK must be set in a unit file such as
+# /etc/systemd/system/elasticsearch.service.d/override.conf.
+#MAX_LOCKED_MEMORY=unlimited
+
+# Maximum number of VMA (Virtual Memory Areas) a process can own
+# When using Systemd, this setting is ignored and the 'vm.max_map_count'
+# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
+#MAX_MAP_COUNT=262144

roles/odfekibana/tasks/main.yml

+---
+
+#- name: Create config directory
+#  file:
+#    name: config
+#    state: directory
+#    mode: 0700
+#  tags:
+#    - start
+
+- name: Copy certificates in odfe kibana conf dir
+  copy:
+    src:  "{{ item }}"
+    dest: "config/{{ item }}"
+    mode: 0600
+  with_items:
+    - "{{ inventory_hostname }}.p12"
+    - "{{ inventory_hostname }}.crt"
+    - "{{ inventory_hostname }}.key"
+    - cacerts.jks
+#    - "{{nifiadmin.0[0]}}.p12"
+  tags:
+    - start
+
+#- name: Configure sysconfig
+#  template:
+#    src: sysconfig_elasticsearch.j2
+#    dest: sysconfig_elasticsearch
+#  tags:
+#    - start
+#
+#- name: Copy sysconfig to /etc
+#  command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch"
+#  tags: 
+#    - start
+
+#  lineinfile:
+#    path: /etc/sysconfig/elasticsearch
+#    regexp: '^ES_PATH_CONF='
+#    line: ES_PATH_CONF=/usr/share/elasticsearch/config
+#  tags:
+#    - start
+- name: Configure odfe kibana properties
+  template:
+    src: "{{item}}.j2"
+    dest: "config/{{item}}"
+  with_items:
+    - kibana.yml
+  tags:
+    - start
+
+- name: Configure odfe kibana start script
+  template:
+    src: "{{item}}.j2"
+    dest: "{{item}}"
+    mode: 0750
+  with_items:
+    - startkibana.sh
+  tags:
+    - start
+
+#- name: Exit here to test ODFE
+#  meta: end_play
+#  tags:
+#    - start
+
+- name: Start OpenDistro Kibana for Elasticsearch
+  command: /usr/share/kibana/startkibana.sh
+  #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml &
+  #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &"
+  tags:
+    - start
+
+- name: Wait for Kibana
+  wait_for:
+    host: "{{groups['odfekibanacontainers'][0]}}"
+    port: 5601
+    state: started
+    delay: 5
+  tags:
+    - start
+
+#- name: check reachable hosts
+#  gather_facts: no
+#  tasks:
+#    - command: ping -c1 {{ inventory_hostname }}
+#      delegate_to: localhost
+#      register: ping_result
+#      ignore_errors: yes
+#    - group_by: key=reachable
+#      when: ping_result|success
+
+#- name: Stop OpenDistro Kibana for Elasticsearch
+#  command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+#  tags:
+#    - stop
+
+- name: Stop OpenDistro Kibana for Elasticsearch
+  command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
+  tags:
+    - stop

roles/odfekibana/templates/kibana.yml.j2

+---
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# Description: 
+# Default Kibana configuration from kibana-docker.
+
+#logging.verbose: true
+cpu.cgroup.path.override: /
+cpuacct.cgroup.path.override: /
+pid.file: {{inventory_hostname}}.pid
+
+server.name: {{inventory_hostname}}
+server.host: "{{inventory_hostname}}"
+#elasticsearch.hosts: https://localhost:9200
+elasticsearch.hosts: https://{{groups['odfeescontainers'][0]}}:9200
+elasticsearch.ssl.verificationMode: none
+elasticsearch.username: kibanaserver
+elasticsearch.password: kibanaserver
+elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"]
+
+opendistro_security.multitenancy.enabled: true
+opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
+opendistro_security.readonly_mode.roles: ["kibana_read_only"]
+
+newsfeed.enabled: false
+telemetry.optIn: false
+telemetry.enabled: false
+
+#opendistro_security.auth.type: "openid"
+#opendistro_security.openid.connect_url: "https://dsldev.gn4-3-wp8-soc.sunet.se:10443/auth/realms/GN43WP8T31SOC1/.well-known/openid-configuration"
+#opendistro_security.openid.client_id: "dsoclab-kibana"
+#opendistro_security.openid.client_secret: "ccaa137f-2a2b-48ae-bcce-9e1fbcbbf181"
+#opendistro_security.openid.root_ca: /usr/share/kibana/config/dslca.crt
+##   opendistro_security.openid.root_ca: /usr/share/kibana/config/gn43wp8t31ca.crt
+#opendistro_security.openid.base_redirect_url: "https://dsldev.gn4-3-wp8-soc.sunet.se:5601"
+
+opendistro_security.cookie.secure: true
+opendistro_security.cookie.password: "MezgW6l2v9BWi6wEwbEn4gaqJZbWGPSI"
+
+server.ssl.enabled: true
+#server.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
+#server.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
+server.ssl.keystore.path: /usr/share/kibana/config/{{inventory_hostname}}.p12
+server.ssl.keystore.password: {{kspass}}
+#server.ssl.certificateAuthorities:
+#server.ssl.truststore.path: jks (p12?)
+#server.ssl.truststore.password:
+
+
+#elasticsearch.ssl.certificate: /usr/share/kibana/config/odfe-kibana.crt
+#elasticsearch.ssl.key: /usr/share/kibana/config/odfe-kibana.key
+#elasticsearch.ssl.certificateAuthorities: /usr/share/kibana/config/dslca.crt
+
+opendistro_security.allow_client_certificates: true

roles/odfekibana/templates/startkibana.sh.j2

+#!/bin/bash -x
+#exec /usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml --verbose > kblog 2>&1 &
+/usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml > kblog 2>&1 &
+# disown
+
+