Merge branch 'opensearch' into 'master'

Migrate from ODFE to OpenSearch

See merge request !5

roles/odfekibana/tasks/stop.yml

----
-
-- name: Stop OpenDistro Kibana for Elasticsearch
-  remote_user: root
-  command: "supervisorctl stop kibana"

roles/odfekibana/tasks/update-config.yml

----
-
-- name: Configure odfe kibana properties
-  remote_user: kibana
-  template:
-    src: "{{item}}.j2"
-    dest: "config/{{item}}"
-  with_items:
-    - kibana.yml
-
-- name: Configure odfe kibana start script
-  remote_user: kibana
-  template:
-    src: "{{item}}.j2"
-    dest: "{{item}}"
-    mode: 0750
-  with_items:
-    - startkibana.sh
-
-- name: Generate configuration for thehive_button plugin
-  remote_user: kibana
-  template:
-    src: files/env.js.j2
-    dest: "/usr/share/kibana/plugins/thehive_button/public/env.js"
-    owner: kibana
-    group: kibana
-
-
-- name: Copy kibana_graphs.ndjson to container
-  remote_user: kibana
-  template:
-    src: "kibana_graphs.ndjson.j2"
-    dest: /tmp/kibana_graphs.ndjson
-

roles/odfekibana/templates/kibana.yml.j2

----
-# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License").
-# You may not use this file except in compliance with the License.
-# A copy of the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is distributed
-# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-# express or implied. See the License for the specific language governing
-# permissions and limitations under the License.
-
-# Description: 
-# Default Kibana configuration from kibana-docker.
-
-#logging.verbose: true
-cpu.cgroup.path.override: /
-cpuacct.cgroup.path.override: /
-pid.file: {{inventory_hostname}}.pid
-
-server.name: {{inventory_hostname}}
-server.host: "{{inventory_hostname}}"
-#elasticsearch.hosts: https://localhost:9200
-elasticsearch.hosts: https://{{groups['odfeescontainers'][0]}}:9200
-elasticsearch.ssl.verificationMode: none
-elasticsearch.username: kibanaserver
-elasticsearch.password: kibanaserver
-elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"]
-
-opendistro_security.multitenancy.enabled: false
-#opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"]
-opendistro_security.readonly_mode.roles: ["kibana_read_only"]
-
-#new in 7.6
-#newsfeed.enabled: false
-#telemetry.optIn: false
-#telemetry.enabled: false
-
-opendistro_security.auth.type: "openid"
-opendistro_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
-opendistro_security.openid.client_id: "soctools-kibana"
-opendistro_security.openid.client_secret: "{{kibanasecret.value}}"
-opendistro_security.openid.root_ca: "/usr/share/kibana/config/ca.crt"
-opendistro_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601"
-
-opendistro_security.cookie.secure: true
-opendistro_security.cookie.password: "{{lookup("password", "{{playbook_dir}}/secrets/passwords/kibana_cookiepassword length=32")}}"
-
-server.ssl.enabled: true
-server.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
-server.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
-#server.ssl.keystore.path: /usr/share/kibana/config/{{inventory_hostname}}.p12
-#server.ssl.keystore.password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
-#server.ssl.certificateAuthorities:
-#server.ssl.truststore.path: jks (p12?)
-#server.ssl.truststore.password:
-
-
-#elasticsearch.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
-#elasticsearch.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
-#elasticsearch.ssl.certificateAuthorities: /usr/share/kibana/config/{{ca_cn}}.crt
-
-opendistro_security.allow_client_certificates: true

roles/odfekibana/templates/startkibana.sh.j2

-#!/bin/bash -x
-#exec /usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml --verbose > kblog 2>&1 &
-/usr/share/kibana/bin/kibana -c /usr/share/kibana/config/kibana.yml > kblog 2>&1 &
-# disown
-
-

roles/odfekibana/tasks/init.yml → roles/opensearch-dashboards/tasks/init.yml

@@ -14,8 +14,8 @@
 - name: Install the Hive Button plugin
   copy:
     src: "/tmp/kibana-plugins-{{kibana_plugins_version}}/thehive_button"
-    dest: "/usr/share/kibana/plugins"
-    owner: kibana
+    dest: "/opt/opensearch-dashboards/plugins"
+    owner: dashboards
     remote_src: yes
 
 - name: Copy cacert to ca-trust dir
@@ -28,8 +28,8 @@
   remote_user: root
   command: "update-ca-trust"
 
-- name: Copy certificates in odfe kibana conf dir
-  remote_user: kibana
+- name: Copy certificates in Opensearch Dashboards conf dir
+  remote_user: dashboards
   copy:
     src:  "{{ item }}"
     dest: "config/"
@@ -42,86 +42,77 @@
     - "{{playbook_dir}}/secrets/CA/ca.crt"
     - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12"
 
-- name: Get openid authkey
-  remote_user: kibana
+- name: Get authkey
   set_fact:
     kibanasecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/kibanasecret',convert_data=False) | from_json }}"
 
-- name: Configure odfe kibana properties
-  remote_user: kibana
+- name: Configure Opensearch Dashboards properties
+  remote_user: dashboards
   template:
     src: "{{item}}.j2"
     dest: "config/{{item}}"
   with_items:
-    - kibana.yml
-
-- name: Configure odfe kibana start script
-  remote_user: kibana
-  template:
-    src: "{{item}}.j2"
-    dest: "{{item}}"
-    mode: 0750
-  with_items:
-    - startkibana.sh
+    - opensearch_dashboards.yml
 
 - name: Generate configuration for thehive_button plugin
-  remote_user: kibana
+  remote_user: dashboards
   template:
     src: files/env.js.j2
-    dest: "/usr/share/kibana/plugins/thehive_button/public/env.js"
-    owner: kibana
-    group: kibana
+    dest: "/opt/opensearch-dashboards/plugins/thehive_button/public/env.js"
+    owner: dashboards
+    group: dashboards
 
 
-- name: Start Kibana
+- name: Start Opensearch Dashboards
   remote_user: root
-  shell: "supervisorctl start kibana"
+  shell: "supervisorctl start opensearch-dashboards"
 
-- name: Wait for Kibana
-  remote_user: kibana
+- name: Wait for Opensearch Dashboards
+  remote_user: dashboards
   wait_for:
-    host: "{{groups['odfekibanacontainers'][0]}}"
+    host: "{{groups['opensearchdashboardscontainers'][0]}}"
     port: 5601
     state: started
     delay: 5
 
-- name: Check Kibana health
-  remote_user: kibana
+- name: Check Opensearch Dashboards health
+  remote_user: dashboards
   shell: 'curl -k -b /tmp/cookie.txt -c /tmp/cookie.txt -X "GET" "https://{{soctoolsproxy}}:5601/api/status" \
-              | egrep status....overall....state...green'
+          --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/opensearches_adminpass")}} \
+              | egrep "status....overall....since...[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}.[0-9]{1,2}.[0-9]{3}Z...state...green"'
   register: result
   until: result.rc == 0
   retries: 90
   delay: 2
   ignore_errors: yes
 
-- name: Copy kibana_graphs.ndjson to container
-  remote_user: kibana
+- name: Copy opensearch-dashboards_graphs.ndjson to container
+  remote_user: dashboards
   template:
-    src: "kibana_graphs.ndjson.j2"
-    dest: /tmp/kibana_graphs.ndjson
+    src: "opensearch-dashboards_graphs.ndjson.j2"
+    dest: /tmp/opensearch-dashboards_graphs.ndjson
 
-- name: Import graphs to kibana
-  remote_user: kibana
+- name: Import graphs to Opensearch Dashboards
+  remote_user: dashboards
   shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/saved_objects/_import?overwrite=true" \
           -b /tmp/cookie.txt -c /tmp/cookie.txt \
-          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \
-          -H "kbn-xsrf: reporting" -H "Content-Type: multipart/form-data" \
-          -F "file=@/tmp/kibana_graphs.ndjson"'
+          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/opensearches_adminpass")}} \
+          -H "osd-xsrf: reporting" -H "Content-Type: multipart/form-data" \
+          -F "file=@/tmp/opensearch-dashboards_graphs.ndjson"'
   ignore_errors: True
 
 - name: Copy role modification json to container
-  remote_user: kibana
+  remote_user: dashboards
   template:
     src: "role.json.j2"
     dest: /tmp/role.json
 
 - name: Grant admin permissions to users
-  remote_user: kibana
+  remote_user: dashboards
   shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/v1/configuration/rolesmapping/all_access" \
           -b /tmp/cookie.txt -c /tmp/cookie.txt \
-          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \
-          -H "kbn-xsrf: reporting" -H "Content-Type: application/json" \
+          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/opensearches_adminpass")}} \
+          -H "osd-xsrf: reporting" -H "Content-Type: application/json" \
           -d @/tmp/role.json'
 
 - name: Set Autostart for supervisord's services

roles/odfekibana/tasks/main.yml → roles/opensearch-dashboards/tasks/main.yml

@@ -6,17 +6,17 @@
 - include: start.yml
   tags:
    - start
-   - start-odfekibana
+   - start-opensearch-dashboards
    - init
 - include: stop.yml
   tags:
    - stop
-   - stop-odfekibana
+   - stop-opensearch-dashboards
 - include: update-config.yml
   tags:
    - update-config
-   - update-odfekibana-config
+   - update-opensearch-dashboards-config
 - include: restart.yml
   tags:
    - restart
-   - restart-odfekibana
+   - restart-opensearch-dashboards

roles/odfekibana/tasks/restart.yml → roles/opensearch-dashboards/tasks/restart.yml

@@ -2,18 +2,18 @@
 
 - name: Restart Kibana
   remote_user: root
-  shell: "supervisorctl restart kibana"
+  shell: "supervisorctl restart opensearch-dashboards"
 
 - name: Wait for Kibana
-  remote_user: kibana
+  remote_user: dashboards
   wait_for:
-    host: "{{groups['odfekibanacontainers'][0]}}"
+    host: "{{groups['opensearchdashboardscontainers'][0]}}"
     port: 5601
     state: started
     delay: 5
 
-- name: Check Kibana health
-  remote_user: kibana
+- name: Check Opensearch Dashboards health
+  remote_user: dashboards
   shell: 'curl -k -b /tmp/cookie.txt -c /tmp/cookie.txt -X "GET" "https://{{soctoolsproxy}}:5601/api/status" \
               | egrep status....overall....state...green'
   register: result

roles/odfekibana/tasks/start.yml → roles/opensearch-dashboards/tasks/start.yml

 ---
 
-- name: Start Kibana
+- name: Start Opensearch Dashboards
   remote_user: root
-  shell: "supervisorctl start kibana"
+  shell: "supervisorctl start opensearch-dashboards"
 
-- name: Wait for Kibana
-  remote_user: kibana
+- name: Wait for Opensearch Dashboards
+  remote_user: dashboards
   wait_for:
-    host: "{{groups['odfekibanacontainers'][0]}}"
+    host: "{{groups['opensearchdashboardscontainers'][0]}}"
     port: 5601
     state: started
     delay: 5
 
-- name: Check Kibana health
-  remote_user: kibana
+- name: Check Opensearch Dashboards health
+  remote_user: dashboards
   shell: 'curl -k -b /tmp/cookie.txt -c /tmp/cookie.txt -X "GET" "https://{{soctoolsproxy}}:5601/api/status" \
-              | egrep status....overall....state...green'
+          --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/opensearches_adminpass")}} \
+              | egrep "status....overall....since...[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}.[0-9]{1,2}.[0-9]{3}Z...state...green"'
   register: result
   until: result.rc == 0
   retries: 90
   delay: 2
   ignore_errors: yes
-

roles/opensearch-dashboards/tasks/stop.yml

+---
+
+- name: Stop Opensearch Dashboards
+  remote_user: root
+  command: "supervisorctl stop opensearch-dashboards"

roles/opensearch-dashboards/tasks/update-config.yml

+---
+
+- name: Get authkey
+  set_fact:
+    kibanasecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/kibanasecret',convert_data=False) | from_json }}"
+
+- name: Configure Opensearch Dashboards properties
+  remote_user: dashboards
+  template:
+    src: "{{item}}.j2"
+    dest: "config/{{item}}"
+  with_items:
+    - opensearch_dashboards.yml
+
+- name: Generate configuration for thehive_button plugin
+  remote_user: dashboards
+  template:
+    src: files/env.js.j2
+    dest: "/opt/opensearch-dashboards/plugins/thehive_button/public/env.js"
+    owner: dashboards
+    group: dashboards
+
+- name: Copy opensearch-dashboards_graphs.ndjson to container
+  remote_user: dashboards
+  template:
+    src: "opensearch-dashboards_graphs.ndjson.j2"
+    dest: /tmp/opensearch-dashboards_graphs.ndjson
+
+- name: Import graphs to Opensearch Dashboards
+  remote_user: dashboards
+  shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/saved_objects/_import?overwrite=true" \
+          -b /tmp/cookie.txt -c /tmp/cookie.txt \
+          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/opensearches_adminpass")}} \
+          -H "osd-xsrf: reporting" -H "Content-Type: multipart/form-data" \
+          -F "file=@/tmp/opensearch-dashboards_graphs.ndjson"'
+  ignore_errors: True

roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2

+---
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# Description:
+# Default configuration for OpenSearch Dashboards
+
+# OpenSearch Dashboards is served by a back end server. This setting specifies the port to use.
+# server.port: 5601
+
+# Specifies the address to which the OpenSearch Dashboards server will bind. IP addresses and host names are both valid values.
+# The default is 'localhost', which usually means remote machines will not be able to connect.
+# To allow connections from remote users, set this parameter to a non-loopback address.
+# server.host: "localhost"
+server.host: "{{inventory_hostname}}"
+
+# Enables you to specify a path to mount OpenSearch Dashboards at if you are running behind a proxy.
+# Use the `server.rewriteBasePath` setting to tell OpenSearch Dashboards if it should remove the basePath
+# from requests it receives, and to prevent a deprecation warning at startup.
+# This setting cannot end in a slash.
+# server.basePath: ""
+
+# Specifies whether OpenSearch Dashboards should rewrite requests that are prefixed with
+# `server.basePath` or require that they are rewritten by your reverse proxy.
+# server.rewriteBasePath: false
+
+# The maximum payload size in bytes for incoming server requests.
+# server.maxPayloadBytes: 1048576
+
+# The OpenSearch Dashboards server's name.  This is used for display purposes.
+# server.name: "your-hostname"
+server.name: {{inventory_hostname}}
+
+# The URLs of the OpenSearch instances to use for all your queries.
+# opensearch.hosts: ["http://localhost:9200"]
+opensearch.hosts: ["https://{{groups['opensearchescontainers'][0]}}:9200","https://{{groups['opensearchescontainers'][1]}}:9200"]
+
+# OpenSearch Dashboards uses an index in OpenSearch to store saved searches, visualizations and
+# dashboards. OpenSearch Dashboards creates a new index if the index doesn't already exist.
+# opensearchDashboards.index: ".opensearch_dashboards"
+
+# The default application to load.
+# opensearchDashboards.defaultAppId: "home"
+
+# Setting for an optimized healthcheck that only uses the local OpenSearch node to do Dashboards healthcheck.
+# This settings should be used for large clusters or for clusters with ingest heavy nodes.
+# It allows Dashboards to only healthcheck using the local OpenSearch node rather than fan out requests across all nodes.
+#
+# It requires the user to create an OpenSearch node attribute with the same name as the value used in the setting
+# This node attribute should assign all nodes of the same cluster an integer value that increments with each new cluster that is spun up
+# e.g. in opensearch.yml file you would set the value to a setting using node.attr.cluster_id:
+# Should only be enabled if there is a corresponding node attribute created in your OpenSearch config that matches the value here
+# opensearch.optimizedHealthcheckId: "cluster_id"
+
+# If your OpenSearch is protected with basic authentication, these settings provide
+# the username and password that the OpenSearch Dashboards server uses to perform maintenance on the OpenSearch Dashboards
+# index at startup. Your OpenSearch Dashboards users still need to authenticate with OpenSearch, which
+# is proxied through the OpenSearch Dashboards server.
+# opensearch.username: "opensearch_dashboards_system"
+# opensearch.password: "pass"
+opensearch.username: kibanaserver
+opensearch.password: kibanaserver
+
+# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
+# These settings enable SSL for outgoing requests from the OpenSearch Dashboards server to the browser.
+# server.ssl.enabled: false
+# server.ssl.certificate: /path/to/your/server.crt
+# server.ssl.key: /path/to/your/server.key
+server.ssl.enabled: true
+server.ssl.key: /opt/opensearch-dashboards/config/{{inventory_hostname}}.key
+server.ssl.certificate: /opt/opensearch-dashboards/config/{{inventory_hostname}}.crt
+
+# Optional settings that provide the paths to the PEM-format SSL certificate and key files.
+# These files are used to verify the identity of OpenSearch Dashboards to OpenSearch and are required when
+# xpack.security.http.ssl.client_authentication in OpenSearch is set to required.
+# opensearch.ssl.certificate: /path/to/your/client.crt
+# opensearch.ssl.key: /path/to/your/client.key
+
+# Optional setting that enables you to specify a path to the PEM file for the certificate
+# authority for your OpenSearch instance.
+opensearch.ssl.certificateAuthorities: [ "/opt/opensearch-dashboards/config/ca.crt" ]
+
+# To disregard the validity of SSL certificates, change this setting's value to 'none'.
+opensearch.ssl.verificationMode: full
+
+# Time in milliseconds to wait for OpenSearch to respond to pings. Defaults to the value of
+# the opensearch.requestTimeout setting.
+# opensearch.pingTimeout: 1500
+
+# Time in milliseconds to wait for responses from the back end or OpenSearch. This value
+# must be a positive integer.
+# opensearch.requestTimeout: 30000
+
+# List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side
+# headers, set this value to [] (an empty list).
+# opensearch.requestHeadersWhitelist: [ authorization ]
+opensearch.requestHeadersWhitelist: [authorization, securitytenant]
+
+# Header names and values that are sent to OpenSearch. Any custom headers cannot be overwritten
+# by client-side headers, regardless of the opensearch.requestHeadersWhitelist configuration.
+# opensearch.customHeaders: {}
+
+# Time in milliseconds for OpenSearch to wait for responses from shards. Set to 0 to disable.
+# opensearch.shardTimeout: 30000
+
+# Logs queries sent to OpenSearch. Requires logging.verbose set to true.
+# opensearch.logQueries: false
+
+# Specifies the path where OpenSearch Dashboards creates the process ID file.
+# pid.file: /var/run/opensearchDashboards.pid
+pid.file: {{inventory_hostname}}.pid
+
+# Enables you to specify a file where OpenSearch Dashboards stores log output.
+# logging.dest: stdout
+
+# Set the value of this setting to true to suppress all logging output.
+# logging.silent: false
+
+# Set the value of this setting to true to suppress all logging output other than error messages.
+# logging.quiet: false
+
+# Set the value of this setting to true to log all events, including system usage information
+# and all requests.
+# logging.verbose: false
+
+# Set the interval in milliseconds to sample system and process performance
+# metrics. Minimum is 100ms. Defaults to 5000.
+# ops.interval: 5000
+
+# Specifies locale to be used for all localizable strings, dates and number formats.
+# Supported languages are the following: English - en , by default , Chinese - zh-CN .
+# i18n.locale: "en"
+
+# Set the allowlist to check input graphite Url. Allowlist is the default check list.
+# vis_type_timeline.graphiteAllowedUrls: ['https://www.hostedgraphite.com/UID/ACCESS_KEY/graphite']
+
+# Set the blocklist to check input graphite Url. Blocklist is an IP list.
+# Below is an example for reference
+# vis_type_timeline.graphiteBlockedIPs: [
+#  //Loopback
+#  '127.0.0.0/8',
+#  '::1/128',
+#  //Link-local Address for IPv6
+#  'fe80::/10',
+#  //Private IP address for IPv4
+#  '10.0.0.0/8',
+#  '172.16.0.0/12',
+#  '192.168.0.0/16',
+#  //Unique local address (ULA)
+#  'fc00::/7',
+#  //Reserved IP address
+#  '0.0.0.0/8',
+#  '100.64.0.0/10',
+#  '192.0.0.0/24',
+#  '192.0.2.0/24',
+#  '198.18.0.0/15',
+#  '192.88.99.0/24',
+#  '198.51.100.0/24',
+#  '203.0.113.0/24',
+#  '224.0.0.0/4',
+#  '240.0.0.0/4',
+#  '255.255.255.255/32',
+#  '::/128',
+#  '2001:db8::/32',
+#  'ff00::/8',
+# ]
+# vis_type_timeline.graphiteBlockedIPs: []
+
+# opensearchDashboards.branding:
+#   logo:
+#     defaultUrl: ""
+#     darkModeUrl: ""
+#   mark:
+#     defaultUrl: ""
+#     darkModeUrl: ""
+#   loadingLogo:
+#     defaultUrl: ""
+#     darkModeUrl: ""
+#   faviconUrl: ""
+#   applicationTitle: ""
+
+# Set the value of this setting to true to capture region blocked warnings and errors
+# for your map rendering services.
+# map.showRegionBlockedWarning: false%
+
+opensearch_security.multitenancy.enabled: false
+#opensearch_security.multitenancy.tenants.preferred: [Private, Global]
+opensearch_security.readonly_mode.roles: [kibana_read_only]
+# Use this setting if you are running opensearch-dashboards without https
+opensearch_security.cookie.secure: true
+opensearch_security.cookie.password: "{{lookup("password", "{{playbook_dir}}/secrets/passwords/opensearch-dashboards_cookiepassword length=32")}}"
+
+opensearch_security.auth.type: "openid"
+opensearch_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
+opensearch_security.openid.client_id: "opensearch-dashboards"
+opensearch_security.openid.client_secret: "{{kibanasecret.value}}"
+opensearch_security.openid.root_ca: "/opt/opensearch-dashboards/config/ca.crt"
+opensearch_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601"