Merge branch 'opensearch' into 'master'

Migrate from ODFE to OpenSearch

See merge request !5

access.ips

@@ -3,7 +3,8 @@
 # For example 0.0.0.0/0 enables access from whole IPv4 internet and ::/0 enables access from whole IPv6
 
 ### HAProxy Stats - Start ###
-#172.22.0.0/16
+0.0.0.0/0
+::/0
 ### HAProxy Stats - End   ###
 
 
@@ -19,9 +20,9 @@
 ### Nifi ports - End   ###
 
 
-### ODFE - Start ###
-#172.22.0.0/16
-### ODFE - End   ###
+### OPENSEARCHES - Start ###
+127.0.0.1
+### OPENSEARCHES - End   ###
 
 
 ### KeyCloak - Start ###
@@ -54,7 +55,7 @@
 ### User Management UI - End   ###
 
 
-### Kibana - Start ###
+### OPENSEARCH-DASHBOARDS - Start ###
 0.0.0.0/0
 ::/0
-#### Kibana - End   ###
+#### OPENSEARCH-DASHBOARDS - End   ###

configure.sh

@@ -11,7 +11,7 @@ wait () {
 	done
 }
 
-echo "By default, all services except HAProxy stats and ODFE are public!"
+echo "By default, all services except HAProxy stats and OPENSEARCH are public!"
 echo "The configuration file: access.ips is used to configure external access to the services"
 echo "Do you want to modify/edit this file now?"
 read -p "(yes|no) [no] : " MODIFY

generate_haproxy_whitelist_files.sh

@@ -2,10 +2,10 @@
 awk '/HAProxy Stats - Start/{flag=1; next} /HAProxy Stats - End/{flag=0} flag' access.ips > roles/haproxy/files/stats_whitelist.lst
 awk '/Nifi Management - Start/{flag=1; next} /Nifi Management - End/{flag=0} flag' access.ips > roles/haproxy/files/nifi_whitelist.lst
 awk '/Nifi ports - Start/{flag=1; next} /Nifi ports - End/{flag=0} flag' access.ips > roles/haproxy/files/nifiports_whitelist.lst
-awk '/ODFE  - Start/{flag=1; next} /ODFE - End/{flag=0} flag' access.ips > roles/haproxy/files/odfe_whitelist.lst
+awk '/OPENSEARCHES - Start/{flag=1; next} /OPENSEARCHES - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearches_whitelist.lst
 awk '/KeyCloak - Start/{flag=1; next} /KeyCloak - End/{flag=0} flag' access.ips > roles/haproxy/files/keycloak_whitelist.lst
 awk '/TheHive - Start/{flag=1; next} /TheHive - End/{flag=0} flag' access.ips > roles/haproxy/files/thehive_whitelist.lst
 awk '/Cortex - Start/{flag=1; next} /Cortex - End/{flag=0} flag' access.ips > roles/haproxy/files/cortex_whitelist.lst
 awk '/MISP - Start/{flag=1; next} /MISP - End/{flag=0} flag' access.ips > roles/haproxy/files/misp_whitelist.lst
 awk '/User Management UI - Start/{flag=1; next} /User Management UI - End/{flag=0} flag' access.ips > roles/haproxy/files/user-mgmt-ui_whitelist.lst
-awk '/Kibana - Start/{flag=1; next} /Kibana - End/{flag=0} flag' access.ips > roles/haproxy/files/kibana_whitelist.lst
+awk '/OPENSEARCH-DASHBOARDS - Start/{flag=1; next} /OPENSEARCH-DASHBOARDS - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearch-dashboards_whitelist.lst

group_vars/all/main.yml

@@ -12,8 +12,6 @@ repo: soctools
 version: 7
 suffix: a20201004
 
-kibana_plugins_version: "v0.7"
-
 THEHIVE_KIBANA_USER:
   username: "kibana"
   name: "Kibana"
@@ -69,19 +67,20 @@ sysctlconfig:
   - { key: "vm.max_map_count" , val:  "524288" }
 
 nifi_javamem: "1500m"
-odfe_javamem: "512m"
+opensearch_javamem: "512m"
 
 nifi_version: 1.12.1
 nifi_repo: "https://archive.apache.org/dist"
 
 ca_cn: "SOCTOOLS-CA"
 
-odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}"
-odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}"
+opensearches_img: "{{repo}}/opensearches:{{version}}{{suffix}}"
+opensearchdashboards_img: "{{repo}}/opensearch-dashboards:{{version}}{{suffix}}"
 #elk_version: "oss-7.6.1"
 elk_version: "oss-7.4.2"
 #odfeplugin_version: "1.7.0.0"
 odfeplugin_version: "1.4.0.0"
+kibana_plugins_version: "v0.7"
 
 openid_realm: "SOCTOOLS1"
 openid_scope: profile
@@ -89,6 +88,7 @@ openid_subjkey: preferred_username
 
 keycloak_img: "{{repo}}/keycloak:{{version}}{{suffix}}"
 
+opensearch_version: "2.3.0"
 elastic_username: "admin"
 
 misp_dbname: "mispdb"
@@ -100,10 +100,9 @@ services:
   - openjdk
   - zookeeper
   - nifi
-  - elasticsearch
-  - kibana
-  - odfees
-  - odfekibana
+  - opensearch
+  - opensearches
+  - opensearch-dashboards
   - keycloak
   - misp
   - cassandra

group_vars/all/variables.template

@@ -21,7 +21,7 @@ soctools_users:
 #    DN: "CN=soc_admin_2"
 #    CN: "soc_admin_2"
     
-# list of users(username) from previous step which will recive admin roles in ODFE. (Minimum one user is required)
+# list of users(username) from previous step which will recive admin roles in OPENSEARCH. (Minimum one user is required)
 ODFE_ADMIN_USERS:
   - soc_admin
 #  -   soc_admin_2

initsoctools.yml

@@ -40,15 +40,15 @@
   roles:
     - cortex
 
-- name: Reconfigure and start OpenDistro for Elasticsearch
-  hosts: odfeescontainers
+- name: Reconfigure and start opensearch
+  hosts: opensearchescontainers
   roles:
-    - odfees
+    - opensearches
 
-- name: Reconfigure and start OpenDistro Kibana for Elasticsearch
-  hosts: odfekibanacontainers
+- name: Reconfigure and start opensearch-dashboards
+  hosts: opensearchdashboardscontainers
   roles:
-    - odfekibana
+    - opensearch-dashboards
 
 - name: Install and run filebeat
   hosts: filebeat

inventories/elasticsearch

-[odfeescontainers]
-soctools-odfe-1 ansible_connection=docker
-soctools-odfe-2 ansible_connection=docker

inventories/filebeat

@@ -3,9 +3,9 @@ soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
 soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
-soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
-soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
-soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/opt/opensearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="opensearch" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/opt/opensearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="opensearch" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/opensearch-dashboards_stdout.log"]' FILEBEAT_LOG_TYPE="osdashboards" FILEBEAT_LOG_FORMAT="json"
 soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
 soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log","/var/opt/rh/rh-mariadb103/lib/mysql/server_audit.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
 soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"

inventories/kibana

-[odfekibanacontainers]
-soctools-kibana ansible_connection=docker

inventories/opensearch

+[opensearchescontainers]
+soctools-opensearch-1 ansible_connection=docker
+soctools-opensearch-2 ansible_connection=docker

inventories/opensearch-dashboards

+[opensearchdashboardscontainers]
+soctools-opensearch-dashboards ansible_connection=docker

restart-soctools.yml

@@ -25,15 +25,15 @@
   roles:
     - nifi
 
-- name: Restart services for OpenDistro for Elasticsearch
-  hosts: odfeescontainers
+- name: Restart services for opensearch
+  hosts: opensearchescontainers
   roles:
-    - odfees
+    - opensearches
 
-- name: Restart services for OpenDistro Kibana for Elasticsearch
-  hosts: odfekibanacontainers
+- name: Restart services for opensearch Kibana
+  hosts: opensearchdashboardscontainers
   roles:
-    - odfekibana
+    - opensearch-dashboards
 
 - name: Restart services for MISP
   hosts: mispcontainers

roles/build/files/odfees/odfesupervisord.conf → roles/build/files/opensearch-dashboards/dashboardssupervisord.conf

@@ -16,18 +16,18 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 [supervisorctl]
 serverurl=unix:///tmp/supervisor.sock
 
-[program:odfe]
-user=elasticsearch
-group=elasticsearch
-directory=/usr/share/elasticsearch
-command=sh -c "/usr/share/elasticsearch/bin/elasticsearch"
+[program:opensearch-dashboards]
+user=dashboards
+group=dashboards
+directory=/opt/opensearch-dashboards
+command=sh -c "/opt/opensearch-dashboards/bin/opensearch-dashboards -c /opt/opensearch-dashboards/config/opensearch_dashboards.yml"
 autostart=false
 autorestart=true
 logfile_maxbytes=10MB
 stdout_logfile_backups = 0
 stderr_logfile_backups = 0
-stderr_logfile = /var/log/supervisor/elasticsearch_stderr.log
-stdout_logfile = /var/log/supervisor/elasticsearch_stdout.log
+stderr_logfile = /var/log/supervisor/opensearch-dashboards_stderr.log
+stdout_logfile = /var/log/supervisor/opensearch-dashboards_stdout.log
 
 [program:filebeat]
 directory=/opt/filebeat

roles/build/files/kibana/kibanasupervisord.conf → roles/build/files/opensearches/opensearchsupervisord.conf

@@ -16,18 +16,18 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 [supervisorctl]
 serverurl=unix:///tmp/supervisor.sock
 
-[program:kibana]
-user=kibana
-group=kibana
-directory=/usr/share/kibana
-command=sh -c "/usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml"
+[program:opensearch]
+user=opensearch
+group=opensearch
+directory=/opt/opensearch
+command=sh -c "/opt/opensearch/bin/opensearch"
 autostart=false
 autorestart=true
 logfile_maxbytes=10MB
 stdout_logfile_backups = 0
 stderr_logfile_backups = 0
-stderr_logfile = /var/log/supervisor/kibana_stderr.log
-stdout_logfile = /var/log/supervisor/kibana_stdout.log
+stderr_logfile = /var/log/supervisor/opensearch_stderr.log
+stdout_logfile = /var/log/supervisor/opensearch_stdout.log
 
 [program:filebeat]
 directory=/opt/filebeat

roles/build/tasks/main.yml

@@ -5,18 +5,16 @@
       - "'CHANGE_ME' not in soctoolsproxy"
     fail_msg: "Review *all* settings in group_vars/all/main.yml"
 
-
-# Create CentOS image if not created yet
+# Create CentOS image if not created yet 
 - name: Check for CentOS image
-  docker_image_info:
-    name: "{{repo}}/centos:{{version}}{{suffix}}"
+  docker_image_info: 
+    name: "{{repo}}/centos:{{version}}{{suffix}}" 
   register: centosimg
 
 - name: Include tasks to create CentOS image
   include_tasks: centos.yml
   when: centosimg.images | length == 0
 
-
 - name: Create main build dir
   file:
     path: "{{docker_build_dir}}"

roles/build/templates/elasticsearch/Dockerfile.j2

-FROM {{repo}}/openjdk:{{version}}{{suffix}}
-
-ENV PATH="/usr/share/elasticsearch/bin:${PATH}"
-
-RUN groupadd -g 1000 elasticsearch && \
-    adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
-
-WORKDIR /usr/share/elasticsearch
-
-RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
-    rpm -Uvh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{elk_version}}-no-jdk-x86_64.rpm && \
-    cp -a /etc/elasticsearch/ /usr/share/elasticsearch/config/ && \
-    chown -R elasticsearch /usr/share/elasticsearch/config && \
-    mkdir -p /usr/share/elasticsearch/data && \
-    chown -R elasticsearch /usr/share/elasticsearch/data && \
-    sed -i -e 's,ES_PATH_CONF=/etc/elasticsearch,ES_PATH_CONF=/usr/share/elasticsearch/config,g' /etc/sysconfig/elasticsearch
-
-RUN echo 'elasticsearch ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
-

roles/build/templates/kibana/Dockerfile.j2

-FROM {{repo}}/centos:{{version}}{{suffix}}
-
-RUN yum install -y supervisor
-RUN yum clean all
-
-ENV PATH="/usr/share/kibana/bin:${PATH}"
-
-RUN groupadd -g 1000 kibana && \
-    adduser -u 1000 -g 1000 -d /usr/share/kibana kibana
-
-WORKDIR /usr/share/kibana
-
-RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
-    rpm -Uvh https://artifacts.elastic.co/downloads/kibana/kibana-{{elk_version}}-x86_64.rpm && \
-    cp -a /etc/kibana/ /usr/share/kibana/config/ && \
-    chown -R kibana /usr/share/kibana/config/
-
-RUN echo 'kibana ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-COPY kibanasupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
-

roles/build/templates/odfees/Dockerfile.j2

-FROM {{repo}}/elasticsearch:{{version}}{{suffix}}
-
-ENV PATH="/usr/share/elasticsearch/bin:${PATH}"
-
-USER root
-WORKDIR /usr/share/elasticsearch
-
-RUN for PLUGIN in \
-    https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-security/opendistro_security-{{odfeplugin_version}}.zip \
-    https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-alerting/opendistro_alerting-{{odfeplugin_version}}.zip \
-    https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-sql/opendistro_sql-{{odfeplugin_version}}.zip; \
-    do bin/elasticsearch-plugin install -b ${PLUGIN}; done && \
-    chown -R elasticsearch plugins/opendistro_security
-
-RUN echo 'elasticsearch ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
-RUN yum install -y supervisor rsync
-RUN yum clean all
-COPY odfesupervisord.conf /etc/supervisord.conf
-ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

roles/build/templates/opensearch-dashboards/Dockerfile.j2

+FROM {{repo}}/centos:{{version}}{{suffix}}
+
+RUN yum install -y supervisor
+RUN yum clean all
+
+ENV PATH="/opt/opensearch-dashboards/bin:${PATH}"
+ARG OPENSEARCH_VERSION={{opensearch_version}}
+
+RUN groupadd -g 1000 dashboards && \
+    adduser -u 1000 -g 1000 -d /opt/opensearch-dashboards -M dashboards
+
+RUN cd /opt && \
+    yum install -y wget sudo  && \
+    wget https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/${OPENSEARCH_VERSION}/opensearch-dashboards-${OPENSEARCH_VERSION}-linux-x64.tar.gz -O /tmp/opensearch-dashboards.tar.gz && \
+    tar -xvzf /tmp/opensearch-dashboards.tar.gz && \
+    ln -s $(find /opt -mindepth 1 -maxdepth 1 -type d | grep -i opensearch) /opt/opensearch-dashboards && \
+    chown -R dashboards:dashboards /opt/opensearch-dashboards/
+
+WORKDIR /opt/opensearch-dashboards
+
+RUN echo 'dashboards ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+COPY dashboardssupervisord.conf /etc/supervisord.conf
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+