replace elasticsearch/odfe/kibana with opensearch and opensearch dashboards

roles/build/templates/opensearch/Dockerfile.j2

+FROM {{repo}}/centos:{{version}}{{suffix}}
+
+ENV PATH="/opt/opensearch/bin:${PATH}"
+ARG OPENSEARCH_VERSION={{opensearch_version}}
+
+RUN groupadd -g 1000 opensearch && \
+    adduser -u 1000 -g 1000 -d /opt/opensearch -M opensearch
+
+
+RUN cd /opt && \
+    yum install -y wget sudo && \
+    wget https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz -O /tmp/opensearch.tar.gz && \
+    tar -xvzf /tmp/opensearch.tar.gz && \
+    ln -s $(find /opt -mindepth 1 -maxdepth 1 -type d | grep -i opensearch) /opt/opensearch && \
+    chown -R opensearch:opensearch /opt/opensearch/
+
+WORKDIR /opt/opensearch
+
+RUN echo 'opensearch ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+
+

roles/build/templates/opensearches/Dockerfile.j2

+FROM {{repo}}/opensearch:{{version}}{{suffix}}
+
+ENV PATH="/opt/opensearch/bin:${PATH}"
+
+USER root
+WORKDIR /opt/opensearch
+
+RUN echo 'opensearch ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
+RUN yum install -y supervisor rsync
+RUN yum clean all
+COPY opensearchsupervisord.conf /etc/supervisord.conf
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

roles/ca/tasks/main.yml

@@ -42,8 +42,8 @@
   command: roles/ca/files/easyrsa/easyrsa show-cert {{item}}
   with_items:
     - "{{ groups['nificontainers'] }}"
-    - "{{ groups['odfeescontainers'] }}"
-    - "{{ groups['odfekibanacontainers'] }}"
+    - "{{ groups['opensearchescontainers'] }}"
+    - "{{ groups['opensearchdashboardscontainers'] }}"
     - "{{ groups['keycloakcontainers'] }}"
     - "{{ groups['mispcontainers'] }}"
     - "{{ groups['thehive'] }}"
@@ -63,8 +63,8 @@
     build-serverClient-full {{item}} nopass
   with_items:
     - "{{ groups['nificontainers'] }}"
-    - "{{ groups['odfeescontainers'] }}"
-    - "{{ groups['odfekibanacontainers'] }}"
+    - "{{ groups['opensearchescontainers'] }}"
+    - "{{ groups['opensearchdashboardscontainers'] }}"
     - "{{ groups['keycloakcontainers'] }}"
     - "{{ groups['mispcontainers'] }}"
     - "{{ groups['thehive'] }}"
@@ -100,8 +100,8 @@
       Enter Export Password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
   with_items:
     - "{{ groups['nificontainers'] }}"
-    - "{{ groups['odfeescontainers'] }}"
-    - "{{ groups['odfekibanacontainers'] }}"
+    - "{{ groups['opensearchescontainers'] }}"
+    - "{{ groups['opensearchdashboardscontainers'] }}"
     - "{{ groups['keycloakcontainers'] }}"
     - "{{ groups['thehive'] }}"
     - "{{ groups['cortex'] }}"

roles/cortex/tasks/configure.yml

@@ -41,7 +41,6 @@
   register: cortexadminuserkey
   args:
     warn: false
-  run_once: True
 
 - set_fact:
     cortexadminuserapikey={{ cortexadminuserkey.stdout }}

roles/cortex/templates/application.conf.j2

@@ -14,7 +14,7 @@ search {
   index = cortex
   # ElasticSearch instance address.
   # For cluster, join address:port with ',': "http://ip1:9200,ip2:9200,ip3:9200"
-  #uri = "https://{{groups['odfeescontainers'][0]}}:9200"
+  #uri = "https://{{groups['opensearchescontainers'][0]}}:9200"
   uri = "http://localhost:9200"
 
   ## Advanced configuration

roles/docker/tasks/main.yml

@@ -6,8 +6,8 @@
 - include: keycloak.yml
 - include: zookeeper.yml
 - include: nifi.yml
-- include: odfees.yml
-- include: odfekibana.yml
+- include: opensearches.yml
+- include: opensearch-dashboards.yml
 - include: misp.yml
 - include: keycloak.yml
 - include: cassandra.yml

roles/docker/tasks/odfekibana.yml → roles/docker/tasks/opensearch-dashboards.yml

 ---
 
-- name: Create odfe kibana containers and connect to network
+- name: Create Opensearch Dashboard containers and connect to network
   docker_container:
     name: "{{ item }}"
     hostname: "{{ item }}"
-    image: "{{ odfekibana_img }}"
+    image: "{{ opensearchdashboards_img }}"
     networks:
       - name: "{{ soctools_netname }}"
     networks_cli_compatible: yes
     interactive: "yes"
-  with_items: "{{ groups['odfekibanacontainers'] }}"
+  with_items: "{{ groups['opensearchdashboardscontainers'] }}"
   tags:
     - start-docker-containers
 
-- name: Disconnect odfe kibana containers from network and remove
+- name: Disconnect Opensearch Dashboard containers from network and remove
   docker_container:
     name: "{{ item }}"
     state: absent
-  with_items: "{{ groups['odfekibanacontainers'] }}"
+  with_items: "{{ groups['opensearchdashboardscontainers'] }}"
   tags:
     - stop-docker-containers
 

roles/docker/tasks/odfees.yml → roles/docker/tasks/opensearches.yml

 ---
 
-- name: Create odfe elasticsearch containers and connect to network
+- name: Create odfe Opensearch containers and connect to network
   docker_container:
     name: "{{ item }}"
     hostname: "{{ item }}"
-    image: "{{ odfees_img }}"
+    image: "{{ opensearches_img }}"
     networks:
       - name: "{{ soctools_netname }}"
     networks_cli_compatible: yes
     volumes:
-      - "{{item}}:/usr/share/elasticsearch/data"
+      - "{{item}}:/opt/opensearch/data"
     interactive: "yes"
-  with_items: "{{ groups['odfeescontainers'] }}"
+  with_items: "{{ groups['opensearchescontainers'] }}"
   tags:
     - start-docker-containers
 
-- name: Disconnect odfe elasticsearch containers from network and remove
+- name: Disconnect Opensearch containers from network and remove
   docker_container:
     name: "{{ item }}"
     state: absent
-  with_items: "{{ groups['odfeescontainers'] }}"
+  with_items: "{{ groups['opensearchescontainers'] }}"
   tags:
     - stop-docker-containers
 

roles/docker/tasks/volumecreate.yml

@@ -32,11 +32,11 @@
   tags:
     - start-docker-containers
 
-- name: Create OpenDistro for Elasticearch volumes
+- name: Create Opensearch volumes
   docker_volume:
     name: "{{item}}"
   with_items:
-    - "{{ groups['odfeescontainers'] }}" 
+    - "{{ groups['opensearchescontainers'] }}" 
   tags:
     - start-docker-containers
 

roles/haproxy/tasks/init.yml

@@ -15,13 +15,13 @@
   - stats
   - nifi
   - nifiports
-  - odfe
+  - opensearch
   - keycloak
   - thehive
   - cortex
   - misp
   - user-mgmt-ui
-  - kibana
+  - opensearch-dashboards
 
 - name: Create required directories
   file:

roles/haproxy/templates/haproxy.cfg.j2

@@ -38,7 +38,7 @@ listen nifiserv
 {% endfor %}
 	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/nifi_whitelist.lst }
 
-listen odfeserv
+listen opensearchserv
         bind *:9200 ssl crt /etc/ssl/haproxy alpn h2,http/1.1
         mode http
         maxconn 5000
@@ -46,10 +46,10 @@ listen odfeserv
         balance source
         option tcpka
         option httplog
-{% for odfehost in groups['odfeescontainers'] %}
-     	server {{odfehost}} {{odfehost}}:9200 ssl check verify none
+{% for opensearchhost in groups['opensearchescontainers'] %}
+     	server {{opensearchhost}} {{opensearchhost}}:9200 ssl check verify none
 {% endfor %}
-	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/odfe_whitelist.lst }
+	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/opensearch_whitelist.lst }
 	
 listen keycloakserv
         bind *:12443 ssl crt /etc/ssl/haproxy alpn h2,http/1.1
@@ -92,7 +92,7 @@ listen cortexserv
 {% endfor %}
 	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/cortex_whitelist.lst }
 
-listen kibanaserv
+listen opensearch-dashboards-serv
         bind *:5601 ssl crt /etc/ssl/haproxy alpn h2,http/1.1
         mode http
         maxconn 5000
@@ -101,10 +101,10 @@ listen kibanaserv
         option tcpka
         option forwardfor
         option httplog
-{% for kibanahost in groups['odfekibanacontainers'] %}
-        server {{kibanahost}} {{kibanahost}}:5601 ssl check verify none
+{% for opensearchdashboardshost in groups['opensearchdashboardscontainers'] %}
+        server {{opensearchdashboardshost}} {{opensearchdashboardshost}}:5601 ssl check verify none
 {% endfor %}
-	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/kibana_whitelist.lst }
+	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/opensearch-dashboards_whitelist.lst }
 
 listen mispserv
         bind *:6443 ssl crt /etc/ssl/haproxy alpn h2,http/1.1

roles/nifi/templates/flow.xml.j2

@@ -12617,8 +12617,8 @@
     <variable name="elastic_username" value="{{ elastic_username }}" />
     <variable name="misp_url" value="{{ misp_url }}" />
     <variable name="elastic_url" value="https://{{ soctoolsproxy }}:9200" />
-    <variable name="elastic_password" value="{{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}" />
+    <variable name="elastic_password" value="{{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}" />
   </rootGroup>
   <controllerServices />
   <reportingTasks />
-</flowController>
+</flowController>
\ No newline at end of file

roles/odfees/tasks/stop.yml

----
-
-- name: Stop OpenDistro for Elasticsearch
-  remote_user: root
-  command: "supervisorctl stop odfe"
-

roles/odfees/templates/config/elasticsearch.yml.j2

-cluster.name: "soctools-cluster"
-#network.host: 0.0.0.0
-network.host: {{ inventory_hostname }}
-discovery.seed_hosts:
-{% for odfees in groups['odfeescontainers'] %}
-  - {{ odfees }}
-{% endfor %}
-#discovery.type: single-node
-transport.port: 9300
-
-path.logs: /usr/share/elasticsearch/logs
-# # minimum_master_nodes need to be explicitly set when bound on a public IP
-# # set to 1 to allow single node clusters
-# # Details: https://github.com/elastic/elasticsearch/pull/17288
-#discovery.zen.minimum_master_nodes: 1
-
-# # Breaking change in 7.0
-# # https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#breaking_70_discovery_changes
-cluster.initial_master_nodes: 
-{% for odfees in groups['odfeescontainers'] %}
-  - {{ odfees }}
-{% endfor %}
-
-#    - elasticsearch1
-#    - docker-test-node-1 
-######## Start OpenDistro for Elasticsearch Security Demo Configuration ########
-# WARNING: revise all the lines below before you go into production
-# opendistro_security.ssl.transport.pemcert_filepath: esnode.pem
-# opendistro_security.ssl.transport.pemkey_filepath: esnode-key.pem
-
-opendistro_security.ssl.transport.keystore_type: pkcs12
-opendistro_security.ssl.transport.keystore_filepath: {{ inventory_hostname }}.p12
-opendistro_security.ssl.transport.keystore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
-#opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
-opendistro_security.ssl.transport.truststore_type: jks
-opendistro_security.ssl.transport.truststore_filepath: cacerts.jks
-opendistro_security.ssl.transport.truststore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
-opendistro_security.ssl.transport.enforce_hostname_verification: false
-
-opendistro_security.ssl.http.enabled: true
-# opendistro_security.ssl.http.pemcert_filepath: esnode.pem
-# opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem
-opendistro_security.ssl.http.keystore_type: pkcs12
-opendistro_security.ssl.http.keystore_filepath: {{ inventory_hostname }}.p12
-opendistro_security.ssl.http.keystore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}"
-opendistro_security.ssl.http.truststore_type: jks
-opendistro_security.ssl.http.truststore_filepath: cacerts.jks
-opendistro_security.ssl.http.truststore_password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
-#opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem
-#opendistro_security.ssl.http.clientauth_mode: optional
-opendistro_security.allow_unsafe_democertificates: false
-opendistro_security.allow_default_init_securityindex: false
-opendistro_security.authcz.admin_dn:
-  - "{{soctools_users[0].DN}}"
-
-opendistro_security.nodes_dn:
-{% for odfees in groups['odfeescontainers'] %}
-  - "CN={{ odfees }}"
-{% endfor %}
-
-opendistro_security.audit.type: internal_elasticsearch
-opendistro_security.enable_snapshot_restore_privilege: true
-opendistro_security.check_snapshot_restore_write_privileges: true
-opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
-cluster.routing.allocation.disk.threshold_enabled: false
-node.max_local_storage_nodes: 3
-######## End OpenDistro for Elasticsearch Security Demo Configuration ########

roles/odfees/templates/config/log4j2.properties.j2

-status = error
-
-appender.console.type = Console
-appender.console.name = console
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
-
-appender.rolling.type = RollingFile
-appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json
-appender.rolling.layout.type = ESJsonLayout
-appender.rolling.layout.type_name = server
-appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz
-appender.rolling.policies.type = Policies
-appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.rolling.policies.time.interval = 1
-appender.rolling.policies.time.modulate = true
-appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.rolling.policies.size.size = 20MB
-appender.rolling.strategy.type = DefaultRolloverStrategy
-appender.rolling.strategy.fileIndex = nomax
-appender.rolling.strategy.action.type = Delete
-appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
-appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
-appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
-appender.rolling.strategy.action.condition.nested_condition.exceeds = 100MB
-
-rootLogger.level = info
-#rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.rolling.ref = rolling

roles/odfees/templates/sysconfig_elasticsearch.j2

-################################
-# Elasticsearch
-################################
-
-# Elasticsearch home directory
-ES_HOME=/usr/share/elasticsearch
-
-# Elasticsearch Java path
-#JAVA_HOME=
-
-# Elasticsearch configuration directory
-ES_PATH_CONF=/usr/share/elasticsearch/config
-
-# Elasticsearch PID directory
-#PID_DIR=/var/run/elasticsearch
-
-# Additional Java OPTS
-#ES_JAVA_OPTS=
-
-# Configure restart on package upgrade (true, every other setting will lead to not restarting)
-#RESTART_ON_UPGRADE=true
-
-################################
-# Elasticsearch service
-################################
-
-# SysV init.d
-#
-# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
-ES_STARTUP_SLEEP_TIME=5
-
-################################
-# System properties
-################################
-
-# Specifies the maximum file descriptor number that can be opened by this process
-# When using Systemd, this setting is ignored and the LimitNOFILE defined in
-# /usr/lib/systemd/system/elasticsearch.service takes precedence
-#MAX_OPEN_FILES=65535
-
-# The maximum number of bytes of memory that may be locked into RAM
-# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
-# in elasticsearch.yml.
-# When using systemd, LimitMEMLOCK must be set in a unit file such as
-# /etc/systemd/system/elasticsearch.service.d/override.conf.
-#MAX_LOCKED_MEMORY=unlimited
-
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-# When using Systemd, this setting is ignored and the 'vm.max_map_count'
-# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
-#MAX_MAP_COUNT=262144

roles/odfekibana/tasks/stop.yml

----
-
-- name: Stop OpenDistro Kibana for Elasticsearch
-  remote_user: root
-  command: "supervisorctl stop kibana"

roles/odfekibana/tasks/update-config.yml

----
-
-- name: Configure odfe kibana properties
-  remote_user: kibana
-  template:
-    src: "{{item}}.j2"
-    dest: "config/{{item}}"
-  with_items:
-    - kibana.yml
-
-- name: Configure odfe kibana start script
-  remote_user: kibana
-  template:
-    src: "{{item}}.j2"
-    dest: "{{item}}"
-    mode: 0750
-  with_items:
-    - startkibana.sh
-
-- name: Generate configuration for thehive_button plugin
-  remote_user: kibana
-  template:
-    src: files/env.js.j2
-    dest: "/usr/share/kibana/plugins/thehive_button/public/env.js"
-    owner: kibana
-    group: kibana
-
-
-- name: Copy kibana_graphs.ndjson to container
-  remote_user: kibana
-  template:
-    src: "kibana_graphs.ndjson.j2"
-    dest: /tmp/kibana_graphs.ndjson
-

roles/odfekibana/templates/kibana.yml.j2

----
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License").
-# You may not use this file except in compliance with the License.
-# A copy of the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is distributed
-# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-# express or implied. See the License for the specific language governing
-# permissions and limitations under the License.
-
-# Description: 
-# Default Kibana configuration from kibana-docker.
-
-#logging.verbose: true
-cpu.cgroup.path.override: /
-cpuacct.cgroup.path.override: /
-pid.file: {{inventory_hostname}}.pid
-
-server.name: {{inventory_hostname}}
-server.host: "{{inventory_hostname}}"
-#elasticsearch.hosts: https://localhost:9200
-elasticsearch.hosts: https://{{groups['odfeescontainers'][0]}}:9200
-elasticsearch.ssl.verificationMode: none
-elasticsearch.username: kibanaserver
-elasticsearch.password: kibanaserver
-elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"]
-
-opendistro_security.multitenancy.enabled: false
-#opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
-opendistro_security.readonly_mode.roles: ["kibana_read_only"]
-
-#new in 7.6
-#newsfeed.enabled: false
-#telemetry.optIn: false
-#telemetry.enabled: false
-
-opendistro_security.auth.type: "openid"
-opendistro_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
-opendistro_security.openid.client_id: "soctools-kibana"
-opendistro_security.openid.client_secret: "{{kibanasecret.value}}"
-opendistro_security.openid.root_ca: "/usr/share/kibana/config/ca.crt"
-opendistro_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601"
-
-opendistro_security.cookie.secure: true
-opendistro_security.cookie.password: "{{lookup("password", "{{playbook_dir}}/secrets/passwords/kibana_cookiepassword length=32")}}"
-
-server.ssl.enabled: true
-server.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
-server.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
-#server.ssl.keystore.path: /usr/share/kibana/config/{{inventory_hostname}}.p12
-#server.ssl.keystore.password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
-#server.ssl.certificateAuthorities:
-#server.ssl.truststore.path: jks (p12?)
-#server.ssl.truststore.password:
-
-
-#elasticsearch.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
-#elasticsearch.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
-#elasticsearch.ssl.certificateAuthorities: /usr/share/kibana/config/{{ca_cn}}.crt
-
-opendistro_security.allow_client_certificates: true