diff --git a/access.ips b/access.ips index 9800b686b19f5efe4dd311e387a504754bfcf7b8..03ba5b6f6468198c1cb999753d12007a716c43a6 100644 --- a/access.ips +++ b/access.ips @@ -3,7 +3,7 @@ # For example 0.0.0.0/0 enables access from whole IPv4 internet and ::/0 enables access from whole IPv6 ### HAProxy Stats - Start ### -#172.22.0.0/16 +127.0.0.1 ### HAProxy Stats - End ### @@ -19,9 +19,9 @@ ### Nifi ports - End ### -### OPENSEARCH - Start ### -#172.22.0.0/16 -### OPENSEARCH - End ### +### OPENSEARCHES - Start ### +127.0.0.1 +### OPENSEARCHES - End ### ### KeyCloak - Start ### @@ -54,7 +54,7 @@ ### User Management UI - End ### -### Kibana - Start ### +### OPENSEARCH-DASHBOARDS - Start ### 0.0.0.0/0 ::/0 -#### Kibana - End ### +#### OPENSEARCH-DASHBOARDS - End ### diff --git a/generate_haproxy_whitelist_files.sh b/generate_haproxy_whitelist_files.sh index a60a2451c43478cdfff7755ad23e8a1c7c100a06..3b30698b3a7c7e107b1732570dd2fe111ba9195a 100755 --- a/generate_haproxy_whitelist_files.sh +++ b/generate_haproxy_whitelist_files.sh @@ -2,10 +2,10 @@ awk '/HAProxy Stats - Start/{flag=1; next} /HAProxy Stats - End/{flag=0} flag' access.ips > roles/haproxy/files/stats_whitelist.lst awk '/Nifi Management - Start/{flag=1; next} /Nifi Management - End/{flag=0} flag' access.ips > roles/haproxy/files/nifi_whitelist.lst awk '/Nifi ports - Start/{flag=1; next} /Nifi ports - End/{flag=0} flag' access.ips > roles/haproxy/files/nifiports_whitelist.lst -awk '/OPENSEARCH - Start/{flag=1; next} /OPENSEARCH - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearch-dashboards_whitelist.lst +awk '/OPENSEARCHES - Start/{flag=1; next} /OPENSEARCHES - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearches_whitelist.lst awk '/KeyCloak - Start/{flag=1; next} /KeyCloak - End/{flag=0} flag' access.ips > roles/haproxy/files/keycloak_whitelist.lst awk '/TheHive - Start/{flag=1; next} /TheHive - End/{flag=0} flag' access.ips > roles/haproxy/files/thehive_whitelist.lst awk '/Cortex - Start/{flag=1; next} /Cortex - End/{flag=0} flag' access.ips > roles/haproxy/files/cortex_whitelist.lst awk '/MISP - Start/{flag=1; next} /MISP - End/{flag=0} flag' access.ips > roles/haproxy/files/misp_whitelist.lst awk '/User Management UI - Start/{flag=1; next} /User Management UI - End/{flag=0} flag' access.ips > roles/haproxy/files/user-mgmt-ui_whitelist.lst -awk '/Kibana - Start/{flag=1; next} /Kibana - End/{flag=0} flag' access.ips > roles/haproxy/files/kibana_whitelist.lst +awk '/OPENSEARCH-DASHBOARDS - Start/{flag=1; next} /OPENSEARCH-DASHBOARDS - End/{flag=0} flag' access.ips > roles/haproxy/files/opensearch-dashboards_whitelist.lst diff --git a/initsoctools.yml b/initsoctools.yml index d0dd9cf97f45dfb418c4c404f3eda09514ba2b26..aee08195d24141861b67e1f0675970dd9f9eacf0 100644 --- a/initsoctools.yml +++ b/initsoctools.yml @@ -45,8 +45,8 @@ roles: - opensearches -- name: Reconfigure and start opensearch Kibana - hosts: opensearchkibanacontainers +- name: Reconfigure and start opensearch-dashboards + hosts: opensearchdashboardscontainers roles: - opensearch-dashboards diff --git a/inventories/filebeat b/inventories/filebeat index 2add349aa63b05076383cb8d04f9ec27a4e1f309..a44ad5a3753a52a57c409be2810d26a7353a6b00 100644 --- a/inventories/filebeat +++ b/inventories/filebeat @@ -3,9 +3,9 @@ soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text" soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text" -soctools-odfe-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json" -soctools-odfe-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json" -soctools-kibana ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json" +soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json" +soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json" +opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/kibana_stdout.log"]' FILEBEAT_LOG_TYPE="kibana" FILEBEAT_LOG_FORMAT="json" soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json" soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log","/var/opt/rh/rh-mariadb103/lib/mysql/server_audit.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text" soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text" diff --git a/roles/build/templates/opensearch/Dockerfile.j2 b/roles/build/templates/opensearch/Dockerfile.j2 index bf9794bcd2bdf59072e34315135711e100903e08..2272b661fb626796b8a81e712bf1cafb89c48556 100644 --- a/roles/build/templates/opensearch/Dockerfile.j2 +++ b/roles/build/templates/opensearch/Dockerfile.j2 @@ -12,6 +12,7 @@ RUN cd /opt && \ wget https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz -O /tmp/opensearch.tar.gz && \ tar -xvzf /tmp/opensearch.tar.gz && \ ln -s $(find /opt -mindepth 1 -maxdepth 1 -type d | grep -i opensearch) /opt/opensearch && \ + mkdir /opt/opensearch/data && \ chown -R opensearch:opensearch /opt/opensearch/ WORKDIR /opt/opensearch diff --git a/roles/docker/tasks/opensearches.yml b/roles/docker/tasks/opensearches.yml index 6057b2f9ab9a6560afbcb189be3a5dc31044aac8..81a910c58bcf399f29d67758455ee26dc1e60103 100644 --- a/roles/docker/tasks/opensearches.yml +++ b/roles/docker/tasks/opensearches.yml @@ -1,6 +1,6 @@ --- -- name: Create odfe Opensearch containers and connect to network +- name: Create Opensearch containers and connect to network docker_container: name: "{{ item }}" hostname: "{{ item }}" diff --git a/roles/haproxy/tasks/init.yml b/roles/haproxy/tasks/init.yml index 7f8b77c00756ab6747443138fc151d0e9bc10c2f..972580ae835e7ec6c8ce6dc78e7296184ebedec7 100644 --- a/roles/haproxy/tasks/init.yml +++ b/roles/haproxy/tasks/init.yml @@ -15,7 +15,7 @@ - stats - nifi - nifiports - - opensearch + - opensearches - keycloak - thehive - cortex diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2 index 5f463b537d171c81dba9ba501a078a52396238ba..5e8c7a839b9dc11c67937e6b90504bf0cafa5e76 100644 --- a/roles/haproxy/templates/haproxy.cfg.j2 +++ b/roles/haproxy/templates/haproxy.cfg.j2 @@ -49,7 +49,7 @@ listen opensearchserv {% for opensearchhost in groups['opensearchescontainers'] %} server {{opensearchhost}} {{opensearchhost}}:9200 ssl check verify none {% endfor %} - tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/opensearch_whitelist.lst } + tcp-request connection reject if !{ src -f /usr/local/etc/haproxy/opensearches_whitelist.lst } listen keycloakserv bind *:12443 ssl crt /etc/ssl/haproxy alpn h2,http/1.1 diff --git a/roles/keycloak/templates/initkeycloakrealm.sh.j2 b/roles/keycloak/templates/initkeycloakrealm.sh.j2 index f114c354ddcf6d65a834c21c6cd6d9ce3977f809..3356862b9166ac2a5d10ff82ec5c083b456dd5dd 100644 --- a/roles/keycloak/templates/initkeycloakrealm.sh.j2 +++ b/roles/keycloak/templates/initkeycloakrealm.sh.j2 @@ -26,7 +26,7 @@ NIFICLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":t kcadm.sh create realms/{{openid_realm}}/clients/${NIFICLIENT}/protocol-mappers/models -b '{"protocol":"openid-connect","config":{"id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","multivalued":"","aggregate.attrs":"","user.attribute":"DN","claim.name":"DN","jsonType.label":"String"},"name":"SendDN","protocolMapper":"oidc-usermodel-attribute-mapper"}' kcadm.sh get realms/{{openid_realm}}/clients/${NIFICLIENT}/client-secret --fields value > /opt/jboss/keycloak/nifisecret -KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-kibana","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:5601","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:5601", "https://{{soctoolsproxy}}:5601/auth/openid/login", "https://{{soctoolsproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }') +KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"opensearch-dashboards","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:5601","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:5601", "https://{{soctoolsproxy}}:5601/auth/openid/login", "https://{{soctoolsproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }') kcadm.sh get realms/{{openid_realm}}/clients/${KIBANACLIENT}/client-secret --fields value > /opt/jboss/keycloak/kibanasecret MISPCLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"soctools-misp","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{soctoolsproxy}}:6443","adminUrl": "","redirectUris": ["https://{{soctoolsproxy}}:6443/users/login/keycloak"],"webOrigins": [], "publicClient": false }') diff --git a/roles/misp/tasks/init.yml b/roles/misp/tasks/init.yml index 30ef788d376047cc0f21789c7cf55c1c695479f3..2e8a77df3d5cbb6f631dd9b9387792b8894b0215 100644 --- a/roles/misp/tasks/init.yml +++ b/roles/misp/tasks/init.yml @@ -57,6 +57,17 @@ - name: Check if database is initialized command: /var/www/MISP/checkdb.sh +- name: Start redis-server + command: "supervisorctl start redis-server" + +- name: update misp database with cake Admin runUpdates + command: /var/www/MISP/app/Console/cake Admin runUpdates + remote_user: apache + args: + chdir: /var/www/MISP + vars: + ansible_remote_tmp: /tmp + - name: Recursively change ownership of a directory file: path: /var/www/MISP diff --git a/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2 b/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2 index 7b389ad6b12c482dbad760c9d47d6d0a0dfb1829..c80e00d3aeed4ac3f98189d3357f8e4a21edf252 100644 --- a/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2 +++ b/roles/opensearch-dashboards/templates/opensearch_dashboards.yml.j2 @@ -193,7 +193,7 @@ opensearch_security.cookie.password: "{{lookup("password", "{{playbook_dir}}/sec opensearch_security.auth.type: "openid" opensearch_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration" -opensearch_security.openid.client_id: "soctools-opensearch-dashboards" +opensearch_security.openid.client_id: "opensearch-dashboards" opensearch_security.openid.client_secret: "{{kibanasecret.value}}" opensearch_security.openid.root_ca: "/opt/opensearch-dashboards/config/ca.crt" opensearch_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601" diff --git a/roles/opensearches/tasks/init.yml b/roles/opensearches/tasks/init.yml index a43335eabf971246ed8a74b865fbdf109199f568..fd8baac194693764bf7caaa61e7d7c2c1e5186c4 100644 --- a/roles/opensearches/tasks/init.yml +++ b/roles/opensearches/tasks/init.yml @@ -25,7 +25,7 @@ remote_user: opensearch template: src: "config/{{item}}.j2" - dest: "config/opensearch-security/{{item}}" + dest: "config/{{item}}" with_items: - opensearch.yml - jvm.options @@ -33,37 +33,32 @@ - name: Change password for admin remote_user: opensearch - command: "OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk bash ./plugins/opensearch-security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}" + shell: export OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk; bash plugins/opensearch-security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}} register: adminhash - set_fact: adminhashpwd: "{{ adminhash.stdout }}" - #adminhashpwd: "{{ hostvars[groups['opensearchescontainers'][0]]['adminhash.stdout'] }}" remote_user: opensearch - name: Change password for cortex remote_user: opensearch - # when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname" - command: "OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_opensearch')}}" + shell: export OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk; bash plugins/opensearch-security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_opensearch')}} register: cortexhash - # when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname" - set_fact: cortexhashpwd: "{{ cortexhash.stdout }}" - #adminhashpwd: "{{ hostvars[groups['opensearchescontainers'][0]]['adminhash.stdout'] }}" remote_user: opensearch - name: Configure opensearch_security properties remote_user: opensearch template: src: "securityconfig/{{item}}.j2" - dest: "plugins/opendistro_security/securityconfig/{{item}}" + dest: "config/opensearch-security/{{item}}" with_items: - internal_users.yml - config.yml - roles_mapping.yml - - name: Start opensearch remote_user: root command: "supervisorctl start opensearch" @@ -76,10 +71,10 @@ state: started delay: 5 -#- name: Configure Opensearch security -# remote_user: opensearch -# command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['opensearchescontainers'][0]}} -cd /usr/share/opensearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/opensearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /usr/share/opensearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster" -# when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname" +- name: Configure Opensearch security + remote_user: opensearch + shell: "export OPENSEARCH_JAVA_HOME=/opt/opensearch/jdk; bash ./plugins/opensearch-security/tools/securityadmin.sh -h {{groups['opensearchescontainers'][0]}} -cd /opt/opensearch/config/opensearch-security/ -ks '/opt/opensearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /opt/opensearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster" + when: "'{{groups['opensearchescontainers'][0]}}' in inventory_hostname" - name: Set Autostart for supervisord's services remote_user: root diff --git a/roles/opensearches/templates/config/jvm.options.j2 b/roles/opensearches/templates/config/jvm.options.j2 index 4e8d95ab5ab8b4885c08d6e0f5f8a9873b9bfe51..d53e318c936a4cdc3cb98dfac670eb8fe710b813 100644 --- a/roles/opensearches/templates/config/jvm.options.j2 +++ b/roles/opensearches/templates/config/jvm.options.j2 @@ -19,8 +19,8 @@ # Xms represents the initial size of total heap space # Xmx represents the maximum size of total heap space --Xms{{openserach_javamem}} --Xmx{{openserach_javamem}} +-Xms{{opensearch_javamem}} +-Xmx{{opensearch_javamem}} ################################################################ ## Expert settings diff --git a/roles/opensearches/templates/config/elasticsearch.yml.j2 b/roles/opensearches/templates/config/opensearch.yml.j2 similarity index 100% rename from roles/opensearches/templates/config/elasticsearch.yml.j2 rename to roles/opensearches/templates/config/opensearch.yml.j2 diff --git a/startsoctools.yml b/startsoctools.yml index a0cc41a02357d71ca487104e15e926c2bf427df6..1f3896491352da60f7b28698426b25042745371e 100644 --- a/startsoctools.yml +++ b/startsoctools.yml @@ -45,7 +45,7 @@ roles: - opensearches -- name: Start Opensearch Kibana +- name: Start Opensearch dashboards hosts: opensearchdashboardscontainers roles: - opensearch-dashboards