cleaned up handling of passwords and certificates.

roles/mysql/tasks/secure.yml

@@ -4,7 +4,7 @@
   mysql_user:
     name: root
     host_all: yes
-    password: "{{mysql_dbrootpass}}"
+    password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}"
   tags:
     - start
   ignore_errors: true
@@ -54,7 +54,7 @@
 # 
 # 
 # 
-# UPDATE mysql.user SET Password=PASSWORD('{{mysql_dbrootpass}}') WHERE User='root';
+# UPDATE mysql.user SET Password=PASSWORD('{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}') WHERE User='root';
 # DELETE FROM mysql.user WHERE User='';
 # DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
 # DROP DATABASE IF EXISTS test;
@@ -64,9 +64,9 @@
 # 
 # 
 # #!/bin/bash -x
-# MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}})
-# #MISPINIT=$(echo "select count(id) from users;" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}})
+# MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}})
+# #MISPINIT=$(echo "select count(id) from users;" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}})
 # if [ ${MISPINIT} == "0" ]; then
-#   cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{misp_dbpass}} {{misp_dbname}}
+#   cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}}
 #   touch /var/www/MISP/dbchecked-$(date +%Y%m%d_%H%M%S)
 # fi

roles/mysql/templates/dotmy.cnf.j2

 [client]
 user=root
-password='{{mysql_dbrootpass}}'
+password='{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}'

roles/mysql/templates/mysql_secure.sql.j2

-UPDATE mysql.user SET Password=PASSWORD('{{mysql_dbrootpass}}') WHERE User='root';
+UPDATE mysql.user SET Password=PASSWORD('{{playbook_dir}}/secrets/passwords/mysql_root')}}') WHERE User='root';
 DELETE FROM mysql.user WHERE User='';
 DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
 DROP DATABASE IF EXISTS test;
@@ -6,7 +6,7 @@ DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%';
 
 CREATE DATABASE IF NOT EXISTS {{misp_dbname}};
 {% for misp_host in groups['mispcontainers'] %}
-GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{misp_dbpass}}';
+GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{playbook_dir}}/secrets/passwords/mysql_misp')}}';
 GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}';
 {% endfor %}
 

roles/nifi/tasks/main.yml

@@ -3,7 +3,7 @@
 - name: Copy cacert to ca-trust dir
   remote_user: root
   copy:
-    src: "files/{{ca_cn}}.crt"
+    src: "{{playbook_dir}}/secrets/CA/ca.crt"
     dest: /etc/pki/ca-trust/source/anchors/ca.crt
   tags:
     - start
@@ -18,10 +18,10 @@
   remote_user: nifi
   copy:
     src:  "{{ item }}"
-    dest: "conf/{{ item }}"
+    dest: "conf/"
   with_items:
-    - "{{ inventory_hostname }}.p12"
-    - cacerts.jks
+    - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12"
+    - "{{playbook_dir}}/secrets/CA/cacerts.jks"
     - common-cacerts.jks
   tags:
     - start
@@ -46,7 +46,7 @@
 - name: Get openid authkey
   remote_user: nifi
   set_fact:
-    nifisecret: "{{lookup('file', 'files/nifisecret',convert_data=False) | from_json }}"
+    nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}"
   tags:
     - start
 

roles/nifi/templates/nifi.properties.j2

@@ -154,11 +154,11 @@ nifi.sensitive.props.additional.keys=
 
 nifi.security.keystore=./conf/{{ inventory_hostname }}.p12
 nifi.security.keystoreType=pkcs12
-nifi.security.keystorePasswd={{ kspass}}
+nifi.security.keystorePasswd={{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
 #nifi.security.keyPasswd=IP7Jgn7amiAYi3LRSRk5LGg3t4zlfh0kEKcAaaoxHDo
 nifi.security.truststore=./conf/cacerts.jks
 nifi.security.truststoreType=jks
-nifi.security.truststorePasswd={{ tspass}}
+nifi.security.truststorePasswd={{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}
 nifi.security.user.authorizer=managed-authorizer
 nifi.security.user.login.identity.provider=
 nifi.security.ocsp.responder.url=

roles/odfees/tasks/main.yml

@@ -3,7 +3,7 @@
 - name: Copy cacert to ca-trust dir
   remote_user: root
   copy:
-    src: "files/{{ca_cn}}.crt"
+    src: "{{playbook_dir}}/secrets/CA/ca.crt"
     dest: /etc/pki/ca-trust/source/anchors/ca.crt
   tags:
     - start
@@ -18,12 +18,12 @@
   remote_user: elasticsearch
   copy:
     src:  "{{ item }}"
-    dest: "config/{{ item }}"
+    dest: "config/"
     mode: 0600
   with_items:
-    - "{{ inventory_hostname }}.p12"
-    - cacerts.jks
-    - "{{soctools_users[0].CN}}.p12"
+    - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12"
+    - "{{playbook_dir}}/secrets/CA/cacerts.jks"
+    - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12"
   tags:
     - start
 
@@ -55,7 +55,7 @@
 
 - name: Change password for admin
   remote_user: elasticsearch
-  command: "bash plugins/opendistro_security/tools/hash.sh -p {{odfees_adminpass}}"
+  command: "bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}"
   register: adminhash
   # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
   tags:
@@ -70,7 +70,7 @@
 
 - name: Change password for cortex
   remote_user: elasticsearch
-  command: "bash plugins/opendistro_security/tools/hash.sh -p {{cortex_odfe_pass}}"
+  command: "bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_odfe')}}"
   register: cortexhash
   # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
   tags:
@@ -118,7 +118,7 @@
 
 - name: Configure OpenDistro security
   remote_user: elasticsearch
-  command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{soctools_users[0].CN}}.p12' -kspass {{soctools_users[0].password}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{tspass}} -cn soctools-cluster"
+  command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster"
   when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname"
   tags:
     - start

roles/odfees/templates/config/elasticsearch.yml.j2

@@ -30,11 +30,11 @@ cluster.initial_master_nodes:
 
 opendistro_security.ssl.transport.keystore_type: pkcs12
 opendistro_security.ssl.transport.keystore_filepath: {{ inventory_hostname }}.p12
-opendistro_security.ssl.transport.keystore_password: {{ kspass }}
+opendistro_security.ssl.transport.keystore_password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
 #opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
 opendistro_security.ssl.transport.truststore_type: jks
 opendistro_security.ssl.transport.truststore_filepath: cacerts.jks
-opendistro_security.ssl.transport.truststore_password: {{ tspass }}
+opendistro_security.ssl.transport.truststore_password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}
 opendistro_security.ssl.transport.enforce_hostname_verification: false
 
 opendistro_security.ssl.http.enabled: true
@@ -42,10 +42,10 @@ opendistro_security.ssl.http.enabled: true
 # opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem
 opendistro_security.ssl.http.keystore_type: pkcs12
 opendistro_security.ssl.http.keystore_filepath: {{ inventory_hostname }}.p12
-opendistro_security.ssl.http.keystore_password: {{ kspass }}
+opendistro_security.ssl.http.keystore_password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
 opendistro_security.ssl.http.truststore_type: jks
 opendistro_security.ssl.http.truststore_filepath: cacerts.jks
-opendistro_security.ssl.http.truststore_password: {{ tspass }}
+opendistro_security.ssl.http.truststore_password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}
 #opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem
 #opendistro_security.ssl.http.clientauth_mode: optional
 opendistro_security.allow_unsafe_democertificates: false

roles/odfekibana/tasks/main.yml

@@ -11,7 +11,7 @@
 - name: Copy cacert to ca-trust dir
   remote_user: root
   copy:
-    src: "files/{{ca_cn}}.crt"
+    src: "{{playbook_dir}}/secrets/CA/ca.crt"
     dest: /etc/pki/ca-trust/source/anchors/ca.crt
   tags:
     - start
@@ -26,22 +26,22 @@
   remote_user: kibana
   copy:
     src:  "{{ item }}"
-    dest: "config/{{ item }}"
+    dest: "config/"
     mode: 0600
   with_items:
-    - "{{ inventory_hostname }}.p12"
-    - "{{ inventory_hostname }}.crt"
-    - "{{ inventory_hostname }}.key"
-    - cacerts.jks
-    - "{{ca_cn}}.crt"
-    - "{{soctools_users[0].CN}}.p12"
+    - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12"
+    - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt"
+    - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key"
+    - "{{playbook_dir}}/secrets/CA/cacerts.jks"
+    - "{{playbook_dir}}/secrets/CA/ca.crt"
+    - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12"
   tags:
     - start
 
 - name: Get openid authkey
   remote_user: kibana
   set_fact:
-    kibanasecret: "{{lookup('file', 'files/kibanasecret',convert_data=False) | from_json }}"
+    kibanasecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/kibanasecret',convert_data=False) | from_json }}"
   tags:
     - start
 
@@ -158,7 +158,7 @@
   remote_user: kibana
   shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/saved_objects/_import?overwrite=true" \
           -b /tmp/cookie.txt -c /tmp/cookie.txt \
-          -k --user admin:{{ odfees_adminpass }} \
+          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \
           -H "kbn-xsrf: reporting" -H "Content-Type: multipart/form-data" \
           -F "file=@/tmp/kibana_graphs.ndjson"'
   tags:
@@ -176,7 +176,7 @@
   remote_user: kibana
   shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/v1/configuration/rolesmapping/all_access" \
           -b /tmp/cookie.txt -c /tmp/cookie.txt \
-          -k --user admin:{{ odfees_adminpass }} \
+          -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \
           -H "kbn-xsrf: reporting" -H "Content-Type: application/json" \
           -d @/tmp/role.json'
   tags:

roles/odfekibana/templates/kibana.yml.j2

@@ -42,7 +42,7 @@ opendistro_security.auth.type: "openid"
 opendistro_security.openid.connect_url: "https://{{soctoolsproxy}}:12443/auth/realms/{{openid_realm}}/.well-known/openid-configuration"
 opendistro_security.openid.client_id: "soctools-kibana"
 opendistro_security.openid.client_secret: "{{kibanasecret.value}}"
-opendistro_security.openid.root_ca: "/usr/share/kibana/config/{{ca_cn}}.crt"
+opendistro_security.openid.root_ca: "/usr/share/kibana/config/ca.crt"
 opendistro_security.openid.base_redirect_url: "https://{{soctoolsproxy}}:5601"
 
 opendistro_security.cookie.secure: true
@@ -52,7 +52,7 @@ server.ssl.enabled: true
 server.ssl.key: /usr/share/kibana/config/{{inventory_hostname}}.key
 server.ssl.certificate: /usr/share/kibana/config/{{inventory_hostname}}.crt
 #server.ssl.keystore.path: /usr/share/kibana/config/{{inventory_hostname}}.p12
-#server.ssl.keystore.password: {{kspass}}
+#server.ssl.keystore.password: {{lookup('password', '{{playbook_dir}}/secrets/passwords/keystore')}}
 #server.ssl.certificateAuthorities:
 #server.ssl.truststore.path: jks (p12?)
 #server.ssl.truststore.password:

utils/flow2template.py

@@ -23,7 +23,10 @@ for v in et.findall(".//variable"):
     elif a['name']=="elastic_username":
         a['value']="{{ elastic_username }}"
     elif a['name']=="elastic_password":
-        a['value']="{{ odfees_adminpass }}"
+        a['value']="{{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}"
+
+for v in et.findall(".//controllerService[name='Soctools CA']/property[name='Truststore Password']/value"):
+    v.text="{{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}}"
     
 et.write(args.templatefile)