From 4900913ceee09771d942b89877259efe92c63f1e Mon Sep 17 00:00:00 2001
From: root <root@srv2.soctools.grena.ge>
Date: Wed, 26 Oct 2022 17:33:01 +0400
Subject: [PATCH] fixes to migrate elasticsearch/odfe/kibana with opensearch
 and opensearch dashboards

---
 access.ips                                    | 12 +++++------
 generate_haproxy_whitelist_files.sh           |  4 ++--
 initsoctools.yml                              |  4 ++--
 inventories/filebeat                          |  6 +++---
 .../build/templates/opensearch/Dockerfile.j2  |  1 +
 roles/docker/tasks/opensearches.yml           |  2 +-
 roles/haproxy/tasks/init.yml                  |  2 +-
 roles/haproxy/templates/haproxy.cfg.j2        |  2 +-
 .../templates/initkeycloakrealm.sh.j2         |  2 +-
 roles/misp/tasks/init.yml                     | 11 ++++++++++
 .../templates/opensearch_dashboards.yml.j2    |  2 +-
 roles/opensearches/tasks/init.yml             | 21 +++++++------------
 .../templates/config/jvm.options.j2           |  4 ++--
 ...elasticsearch.yml.j2 => opensearch.yml.j2} |  0
 startsoctools.yml                             |  2 +-
 15 files changed, 41 insertions(+), 34 deletions(-)
 rename roles/opensearches/templates/config/{elasticsearch.yml.j2 => opensearch.yml.j2} (100%)

diff --git a/access.ips b/access.ips
index 9800b68..03ba5b6 100644
--- a/access.ips
+++ b/access.ips
@@ -3,7 +3,7 @@
 # For example 0.0.0.0/0 enables access from whole IPv4 internet and ::/0 enables access from whole IPv6
 
 ### HAProxy Stats - Start ###
-#172.22.0.0/16
+127.0.0.1
 ### HAProxy Stats - End   ###
 
 
@@ -19,9 +19,9 @@
 ### Nifi ports - End   ###
 
 
-### OPENSEARCH - Start ###
-#172.22.0.0/16
-### OPENSEARCH - End   ###
+### OPENSEARCHES - Start ###
+127.0.0.1
+### OPENSEARCHES - End   ###
 
 
 ### KeyCloak - Start ###
@@ -54,7 +54,7 @@
 ### User Management UI - End   ###
 
 
-### Kibana - Start ###
+### OPENSEARCH-DASHBOARDS - Start ###
 0.0.0.0/0
 ::/0
-#### Kibana - End   ###
+#### OPENSEARCH-DASHBOARDS - End   ###
diff --git a/generate_haproxy_whitelist_files.sh b/generate_haproxy_whitelist_files.sh
index a60a245..3b30698 100755
--- a/generate_haproxy_whitelist_files.sh
+++ b/generate_haproxy_whitelist_files.sh
@@ -2,10 +2,10 @@
 awk '/HAProxy Stats - Start/{flag=1; next} /HAProxy Stats - End/{flag=0} flag' access.ips > roles/haproxy/files/stats_whitelist.lst
 awk '/Nifi Management - Start/{flag=1; next} /Nifi Management - End/{flag=0} flag' access.ips > roles/haproxy/files/nifi_whitelist.lst
 awk '/Nifi ports - Start/{flag=1; next} /Nifi ports - End/{flag=0} flag' access.ips > roles/haproxy/files/nifiports_whitelist.lst
-awk '/OPENSEARCH  - Start/{flag=1; next} /OPENSEARCH - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearch-dashboards_whitelist.lst
+awk '/OPENSEARCHES  - Start/{flag=1; next} /OPENSEARCHES - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearches_whitelist.lst
 awk '/KeyCloak - Start/{flag=1; next} /KeyCloak - End/{flag=0} flag' access.ips > roles/haproxy/files/keycloak_whitelist.lst
 awk '/TheHive - Start/{flag=1; next} /TheHive - End/{flag=0} flag' access.ips > roles/haproxy/files/thehive_whitelist.lst
 awk '/Cortex - Start/{flag=1; next} /Cortex - End/{flag=0} flag' access.ips > roles/haproxy/files/cortex_whitelist.lst
 awk '/MISP - Start/{flag=1; next} /MISP - End/{flag=0} flag' access.ips > roles/haproxy/files/misp_whitelist.lst
 awk '/User Management UI - Start/{flag=1; next} /User Management UI - End/{flag=0} flag' access.ips > roles/haproxy/files/user-mgmt-ui_whitelist.lst
-awk '/Kibana - Start/{flag=1; next} /Kibana - End/{flag=0} flag' access.ips > roles/haproxy/files/kibana_whitelist.lst
+awk '/OPENSEARCH-DASHBOARDS - Start/{flag=1; next} /OPENSEARCH-DASHBOARDS - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearch-dashboards_whitelist.lst
diff --git a/initsoctools.yml b/initsoctools.yml
index d0dd9cf..aee0819 100644
--- a/initsoctools.yml
+++ b/initsoctools.yml
@@ -45,8 +45,8 @@
   roles:
     - opensearches
 
-- name: Reconfigure and start opensearch Kibana
-  hosts: opensearchkibanacontainers
+- name: Reconfigure and start opensearch-dashboards
+  hosts: opensearchdashboardscontainers
   roles:
     - opensearch-dashboards
 
diff --git a/inventories/filebeat b/inventories/filebeat
index 2add349..a44ad5a 100644
--- a/inventories/filebeat
+++ b/inventories/filebeat
@@ -3,9 +3,9 @@ soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
 soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
-soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
-soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
-soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
+opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
 soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
 soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log","/var/opt/rh/rh-mariadb103/lib/mysql/server_audit.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
 soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
diff --git a/roles/build/templates/opensearch/Dockerfile.j2 b/roles/build/templates/opensearch/Dockerfile.j2
index bf9794b..2272b66 100644
--- a/roles/build/templates/opensearch/Dockerfile.j2
+++ b/roles/build/templates/opensearch/Dockerfile.j2
@@ -12,6 +12,7 @@ RUN cd /opt && \
     wget https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz -O /tmp/opensearch.tar.gz && \
     tar -xvzf /tmp/opensearch.tar.gz && \
     ln -s $(find /opt -mindepth 1 -maxdepth 1 -type d | grep -i opensearch) /opt/opensearch && \
+    mkdir /opt/opensearch/data && \
     chown -R opensearch:opensearch /opt/opensearch/
 
 WORKDIR /opt/opensearch
diff --git a/roles/docker/tasks/opensearches.yml b/roles/docker/tasks/opensearches.yml
index 6057b2f..81a910c 100644
--- a/roles/docker/tasks/opensearches.yml
+++ b/roles/docker/tasks/opensearches.yml
@@ -1,6 +1,6 @@
 ---
 
-- name: Create odfe Opensearch containers and connect to network
+- name: Create Opensearch containers and connect to network
   docker_container:
     name: "{{ item }}"
     hostname: "{{ item }}"
diff --git a/roles/haproxy/tasks/init.yml b/roles/haproxy/tasks/init.yml
index 7f8b77c..972580a 100644
--- a/roles/haproxy/tasks/init.yml
+++ b/roles/haproxy/tasks/init.yml
@@ -15,7 +15,7 @@
   - stats
   - nifi
   - nifiports
-  - opensearch
+  - opensearches
   - keycloak
   - thehive
   - cortex
diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2
index 5f463b5..5e8c7a8 100644
--- a/roles/haproxy/templates/haproxy.cfg.j2
+++ b/roles/haproxy/templates/haproxy.cfg.j2
@@ -49,7 +49,7 @@ listen opensearchserv
 {% for opensearchhost in groups['opensearchescontainers'] %}
      	server {{opensearchhost}} {{opensearchhost}}:9200 ssl check verify none
 {% endfor %}
-	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/opensearch_whitelist.lst }
+	tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/opensearches_whitelist.lst }
 	
 listen keycloakserv
         bind *:12443 ssl crt /etc/ssl/haproxy alpn h2,http/1.1
diff --git a/roles/keycloak/templates/initkeycloakrealm.sh.j2 b/roles/keycloak/templates/initkeycloakrealm.sh.j2
index f114c35..3356862 100644
--- a/roles/keycloak/templates/initkeycloakrealm.sh.j2
+++ b/roles/keycloak/templates/initkeycloakrealm.sh.j2
@@ -26,7 +26,7 @@ NIFICLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":t
 kcadm.sh create realms/{{openid_realm}}/clients/${NIFICLIENT}/protocol-mappers/models -b '{"protocol":"openid-connect","config":{"id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","multivalued":"","aggregate.attrs":"","user.attribute":"DN","claim.name":"DN","jsonType.label":"String"},"name":"SendDN","protocolMapper":"oidc-usermodel-attribute-mapper"}'
 kcadm.sh get realms/{{openid_realm}}/clients/${NIFICLIENT}/client-secret --fields value > /opt/jboss/keycloak/nifisecret
 
-KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-kibana","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:5601","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:5601", "https://{{soctoolsproxy}}:5601/auth/openid/login", "https://{{soctoolsproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }')
+KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"opensearch-dashboards","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:5601","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:5601", "https://{{soctoolsproxy}}:5601/auth/openid/login", "https://{{soctoolsproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }')
 kcadm.sh get realms/{{openid_realm}}/clients/${KIBANACLIENT}/client-secret --fields value > /opt/jboss/keycloak/kibanasecret
 
 MISPCLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-misp","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:6443","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:6443/users/login/keycloak"],"webOrigins": [], "publicClient": false }')
diff --git a/roles/misp/tasks/init.yml b/roles/misp/tasks/init.yml
index 30ef788..2e8a77d 100644
--- a/roles/misp/tasks/init.yml
+++ b/roles/misp/tasks/init.yml
@@ -57,6 +57,17 @@
 - name: Check if database is initialized
   command: /var/www/MISP/checkdb.sh
 
+- name: Start redis-server
+  command: "supervisorctl start redis-server"
+
+- name: update misp database with cake Admin runUpdates
+  command: /var/www/MISP/app/Console/cake Admin runUpdates
+  remote_user: apache
+  args:
+    chdir: /var/www/MISP
+  vars:
+    ansible_remote_tmp: /tmp
+
 - name: Recursively change ownership of a directory
   file:
     path: /var/www/MISP
diff --git a/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2 b/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2
index 7b389ad..c80e00d 100644
--- a/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2
+++ b/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2
@@ -193,7 +193,7 @@ opensearch_security.cookie.password: "{{lookup("password", "{{playbook_dir}}/sec
 
 opensearch_security.auth.type: "openid"
 opensearch_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
-opensearch_security.openid.client_id: "soctools-opensearch-dashboards"
+opensearch_security.openid.client_id: "opensearch-dashboards"
 opensearch_security.openid.client_secret: "{{kibanasecret.value}}"
 opensearch_security.openid.root_ca: "/opt/opensearch-dashboards/config/ca.crt"
 opensearch_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601"
diff --git a/roles/opensearches/tasks/init.yml b/roles/opensearches/tasks/init.yml
index a43335e..fd8baac 100644
--- a/roles/opensearches/tasks/init.yml
+++ b/roles/opensearches/tasks/init.yml
@@ -25,7 +25,7 @@
   remote_user: opensearch
   template:
     src: "config/{{item}}.j2"
-    dest: "config/opensearch-security/{{item}}"
+    dest: "config/{{item}}"
   with_items:
     - opensearch.yml
     - jvm.options
@@ -33,37 +33,32 @@
 
 - name: Change password for admin
   remote_user: opensearch
-  command: "OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk bash ./plugins/opensearch-security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}"
+  shell: export OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk; bash plugins/opensearch-security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}
   register: adminhash
 
 - set_fact:
     adminhashpwd: "{{ adminhash.stdout }}"
-    #adminhashpwd: "{{ hostvars[groups['opensearchescontainers'][0]]['adminhash.stdout'] }}"
   remote_user: opensearch
 
 - name: Change password for cortex
   remote_user: opensearch
-  # when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname"
-  command: "OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_opensearch')}}"
+  shell: export OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk; bash plugins/opensearch-security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_opensearch')}}
   register: cortexhash
-  # when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname"
 
 - set_fact:
     cortexhashpwd: "{{ cortexhash.stdout }}"
-    #adminhashpwd: "{{ hostvars[groups['opensearchescontainers'][0]]['adminhash.stdout'] }}"
   remote_user: opensearch
 
 - name: Configure opensearch_security properties
   remote_user: opensearch
   template:
     src: "securityconfig/{{item}}.j2"
-    dest: "plugins/opendistro_security/securityconfig/{{item}}"
+    dest: "config/opensearch-security/{{item}}"
   with_items:
     - internal_users.yml
     - config.yml
     - roles_mapping.yml
 
-
 - name: Start opensearch
   remote_user: root
   command: "supervisorctl start opensearch"
@@ -76,10 +71,10 @@
     state: started
     delay: 5
 
-#- name: Configure Opensearch security
-#  remote_user: opensearch
-#  command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['opensearchescontainers'][0]}} -cd /usr/share/opensearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/opensearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /usr/share/opensearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster"
-#  when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname"
+- name: Configure Opensearch security
+  remote_user: opensearch
+  shell: "export OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk; bash ./plugins/opensearch-security/tools/securityadmin.sh -h {{groups['opensearchescontainers'][0]}} -cd /opt/opensearch/config/opensearch-security/ -ks '/opt/opensearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /opt/opensearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster"
+  when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname"
 
 - name: Set Autostart for supervisord's services
   remote_user: root
diff --git a/roles/opensearches/templates/config/jvm.options.j2 b/roles/opensearches/templates/config/jvm.options.j2
index 4e8d95a..d53e318 100644
--- a/roles/opensearches/templates/config/jvm.options.j2
+++ b/roles/opensearches/templates/config/jvm.options.j2
@@ -19,8 +19,8 @@
 # Xms represents the initial size of total heap space
 # Xmx represents the maximum size of total heap space
 
--Xms{{openserach_javamem}}
--Xmx{{openserach_javamem}}
+-Xms{{opensearch_javamem}}
+-Xmx{{opensearch_javamem}}
 
 ################################################################
 ## Expert settings
diff --git a/roles/opensearches/templates/config/elasticsearch.yml.j2 b/roles/opensearches/templates/config/opensearch.yml.j2
similarity index 100%
rename from roles/opensearches/templates/config/elasticsearch.yml.j2
rename to roles/opensearches/templates/config/opensearch.yml.j2
diff --git a/startsoctools.yml b/startsoctools.yml
index a0cc41a..1f38964 100644
--- a/startsoctools.yml
+++ b/startsoctools.yml
@@ -45,7 +45,7 @@
   roles:
     - opensearches
 
-- name: Start Opensearch Kibana
+- name: Start Opensearch dashboards
   hosts: opensearchdashboardscontainers
   roles:
     - opensearch-dashboards
-- 
GitLab