diff --git a/dockercontainers.yml b/dockercontainers.yml new file mode 100644 index 0000000000000000000000000000000000000000..de07583d50a760fe6f30aed0928f59dca3140e7d --- /dev/null +++ b/dockercontainers.yml @@ -0,0 +1,7 @@ +--- + +- name: Manage docker containers + hosts: soctoolsmain + roles: + - docker + diff --git a/initsoctools.yml b/initsoctools.yml new file mode 100644 index 0000000000000000000000000000000000000000..b4371504f088735a35d38c7fd317456a65c51c51 --- /dev/null +++ b/initsoctools.yml @@ -0,0 +1,57 @@ +--- + +- name: Reconfigure and start haproxy + hosts: haproxy + roles: + - haproxy + +- name: Reconfigure and start mysql + hosts: mysql + roles: + - mysql + +- name: Reconfigure and start Cassandra + hosts: cassandra + roles: + - cassandra + +- name: Reconfigure and start Keycloak + hosts: keycloakcontainers + roles: + - keycloak + +- name: Reconfigure and start MISP + hosts: mispcontainers + roles: + - misp + +- name: Reconfigure and start NiFi + hosts: nificontainers + roles: + - nifi + +- name: Reconfigure and start TheHive + hosts: thehive + roles: + - thehive + +- name: Reconfigure and start Cortex + hosts: cortex + roles: + - cortex + +- name: Reconfigure and start OpenDistro for Elasticsearch + hosts: odfeescontainers + roles: + - odfees + +- name: Reconfigure and start OpenDistro Kibana for Elasticsearch + hosts: odfekibanacontainers + roles: + - odfekibana + +- name: Install and run filebeat + hosts: filebeat + roles: + - filebeat + diff --git a/roles/cassandra/tasks/init.yml b/roles/cassandra/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..e8a1d35d95a27165d5ad85b99043be3ba9fb2fe1 --- /dev/null +++ b/roles/cassandra/tasks/init.yml @@ -0,0 +1,15 @@ +--- + +- name: Configure Cassandra + remote_user: cassandra + template: + src: cassandra.yaml.j2 + dest: /usr/share/cassandra/conf/cassandra.yaml + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + + diff --git a/roles/cassandra/tasks/main.yml b/roles/cassandra/tasks/main.yml index f26e13b0d2279484dea86aa46ee76cb1b8088865..0569ae7800b7889e9d7693c68246aa86cbd7b5f3 100644 --- a/roles/cassandra/tasks/main.yml +++ b/roles/cassandra/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-cassandra + - init - include: stop.yml tags: - stop diff --git a/roles/cassandra/tasks/restart.yml b/roles/cassandra/tasks/restart.yml index b6e575bc1e4e2e1ebd6469bf05165ef0ba84a376..41c4a118022c2a5d205331e433b0156c6cbe7447 100644 --- a/roles/cassandra/tasks/restart.yml +++ b/roles/cassandra/tasks/restart.yml @@ -4,3 +4,10 @@ remote_user: root command: "supervisorctl restart cassandra" +- name: Wait for Cassandra + remote_user: root + wait_for: + host: "{{groups['cassandra'][0]}}" + port: 9042 + state: started + delay: 5 diff --git a/roles/cassandra/tasks/start.yml b/roles/cassandra/tasks/start.yml index 1018944237cac5bfa616169d058ad0db794ea9f4..b3d1cc88f40ca4f81c1e45b7bc55d9b66410930d 100644 --- a/roles/cassandra/tasks/start.yml +++ b/roles/cassandra/tasks/start.yml @@ -1,11 +1,5 @@ --- -- name: Configure Cassandra - remote_user: cassandra - template: - src: cassandra.yaml.j2 - dest: /usr/share/cassandra/conf/cassandra.yaml - - name: Start Cassandra remote_user: root command: "supervisorctl start cassandra" @@ -18,10 +12,4 @@ state: started delay: 5 -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' - diff --git a/roles/cassandra/tasks/update-config.yml b/roles/cassandra/tasks/update-config.yml index ceb0b6003da6268a734cc15461ed264fb5e38485..977839a1075f19c4adee8670e913719bc1c93e44 100644 --- a/roles/cassandra/tasks/update-config.yml +++ b/roles/cassandra/tasks/update-config.yml @@ -6,10 +6,3 @@ src: cassandra.yaml.j2 dest: /usr/share/cassandra/conf/cassandra.yaml -- name: Wait for Cassandra - remote_user: root - wait_for: - host: "{{groups['cassandra'][0]}}" - port: 9042 - state: started - delay: 5 diff --git a/roles/cortex/tasks/init.yml b/roles/cortex/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..c4355124b1c1a6fa3b73c633eb9e9c1fe85d5ddf --- /dev/null +++ b/roles/cortex/tasks/init.yml @@ -0,0 +1,47 @@ +--- + +- name: Copy cacert to ca-trust dir + remote_user: root + copy: + src: "{{playbook_dir}}/secrets/CA/ca.crt" + dest: /etc/pki/ca-trust/source/anchors/ca.crt + +- name: Install cacert to root truststore + remote_user: root + command: "update-ca-trust" + +- name: Copy certificates in cortex conf dir + remote_user: cortex + copy: + src: "{{ item }}" + dest: "/etc/cortex/" + mode: 0600 + with_items: + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" + - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" + - "{{playbook_dir}}/secrets/CA/cacerts.jks" + - "{{playbook_dir}}/secrets/CA/ca.crt" + +- name: Get openid authkey + set_fact: + cortexsecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/cortexsecret',convert_data=False) | from_json}}" + +- name: Configure embedded Elasticsearch 6 + remote_user: root + template: + src: jvm.options.j2 + dest: /etc/elasticsearch/jvm.options + +- name: Configure Cortex + remote_user: cortex + template: + src: application.conf.j2 + dest: /etc/cortex/application.conf + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + diff --git a/roles/cortex/tasks/main.yml b/roles/cortex/tasks/main.yml index 5216d2c717a180540106714d58ae7d21241e4554..a494e3f9b8d4099da6d96ed71d9244b1f516dd81 100644 --- a/roles/cortex/tasks/main.yml +++ b/roles/cortex/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-cortex + - init - include: stop.yml tags: - stop diff --git a/roles/cortex/tasks/start.yml b/roles/cortex/tasks/start.yml index 9b7eb794916702247eea35118caf352bb8895258..347856f1f45fb990bca4ec99511cbcecdce169a6 100644 --- a/roles/cortex/tasks/start.yml +++ b/roles/cortex/tasks/start.yml @@ -1,48 +1,9 @@ --- -- name: Copy cacert to ca-trust dir - remote_user: root - copy: - src: "{{playbook_dir}}/secrets/CA/ca.crt" - dest: /etc/pki/ca-trust/source/anchors/ca.crt - -- name: Install cacert to root truststore - remote_user: root - command: "update-ca-trust" - -- name: Copy certificates in cortex conf dir - remote_user: cortex - copy: - src: "{{ item }}" - dest: "/etc/cortex/" - mode: 0600 - with_items: - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" - - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" - - "{{playbook_dir}}/secrets/CA/cacerts.jks" - - "{{playbook_dir}}/secrets/CA/ca.crt" - -- name: Get openid authkey - set_fact: - cortexsecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/cortexsecret',convert_data=False) | from_json}}" - -- name: Configure embedded Elasticsearch 6 - remote_user: root - template: - src: jvm.options.j2 - dest: /etc/elasticsearch/jvm.options - - name: Start embedded Elasticsearch 6 remote_user: root command: "supervisorctl start elasticsearch" -- name: Configure Cortex - remote_user: cortex - template: - src: application.conf.j2 - dest: /etc/cortex/application.conf - - name: Start Cortex remote_user: root command: "supervisorctl start cortex" @@ -55,9 +16,3 @@ state: started delay: 5 -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' - diff --git a/roles/docker/tasks/cassandra.yml b/roles/docker/tasks/cassandra.yml index b913425d7171f5bb30f9b5a2434b521a09879ae6..bfb41cc51c5cde37a70932bb73b7258f3f90fa77 100644 --- a/roles/docker/tasks/cassandra.yml +++ b/roles/docker/tasks/cassandra.yml @@ -13,7 +13,7 @@ interactive: "yes" with_items: "{{ groups['cassandra'] }}" tags: - - start + - start-docker-containers - name: Disconnect cassandra containers from network and remove docker_container: @@ -21,5 +21,5 @@ state: absent with_items: "{{ groups['cassandra'] }}" tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/cortex.yml b/roles/docker/tasks/cortex.yml index 15a0732d1d90e4403f020e1edf868c34ab0d5dfc..ec2ff382474e80492ca4a119b75413dc63f6b47b 100644 --- a/roles/docker/tasks/cortex.yml +++ b/roles/docker/tasks/cortex.yml @@ -8,15 +8,12 @@ networks: - name: "{{ soctools_netname }}" networks_cli_compatible: yes -# published_ports: -# - "9001:9001" volumes: - "{{item}}:/var/lib/elasticsearch/" interactive: "yes" with_items: "{{ groups['cortex'] }}" tags: - - start - - startcortex + - start-docker-containers - name: Disconnect cortex containers from network and remove docker_container: @@ -24,5 +21,4 @@ state: absent with_items: "{{ groups['cortex'] }}" tags: - - stop - - stopcortex + - stop-docker-containers diff --git a/roles/docker/tasks/haproxy.yml b/roles/docker/tasks/haproxy.yml index b0180761f4f3c18982f21d9f5b8307066146daed..5fb181e93d2ca094431d4d7e7b5a1835d52c2533 100644 --- a/roles/docker/tasks/haproxy.yml +++ b/roles/docker/tasks/haproxy.yml @@ -21,11 +21,11 @@ - "9001:9001" interactive: "yes" tags: - - start + - start-docker-containers - name: Destroy haproxy container docker_container: name: "{{ haproxy_name }}" state: absent tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/keycloak.yml b/roles/docker/tasks/keycloak.yml index 1fddf2a13aa6086a09ef7ad9c0f8ea1c7aa7b3b9..dfc073cbc62c75d2605d51202b7285d9b71c02e3 100644 --- a/roles/docker/tasks/keycloak.yml +++ b/roles/docker/tasks/keycloak.yml @@ -13,7 +13,7 @@ interactive: "yes" with_items: "{{ groups['keycloakcontainers'] }}" tags: - - start + - start-docker-containers - name: Disconnect keycloak containers from network and remove docker_container: @@ -21,5 +21,5 @@ state: absent with_items: "{{ groups['keycloakcontainers'] }}" tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/misp.yml b/roles/docker/tasks/misp.yml index f32e440eeff7e9d8c1406cf76bafde97aaba19d0..953f3869b88613156415ed87ad02f9b54236a0d7 100644 --- a/roles/docker/tasks/misp.yml +++ b/roles/docker/tasks/misp.yml @@ -12,11 +12,11 @@ published_ports: - "6443:6443" tags: - - start + - start-docker-containers - name: Destroy misp container docker_container: name: "{{ misp_name }}" state: absent tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/mysql.yml b/roles/docker/tasks/mysql.yml index ada6fb4843a63b3f8308b1e7eafef51d6da72f02..d7779da3f7ae3b3ecd514d6dd705a29a4f3fd98c 100644 --- a/roles/docker/tasks/mysql.yml +++ b/roles/docker/tasks/mysql.yml @@ -11,11 +11,11 @@ volumes: - "{{mysql_name}}:/var/opt/rh/rh-mariadb103/lib/mysql" tags: - - start + - start-docker-containers - name: Destroy mysql container docker_container: name: "{{ mysql_name }}" state: absent tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/networkcreate.yml b/roles/docker/tasks/networkcreate.yml index 3e1cdf67676f71c9e9efcd8b7419d2d4d25a096a..02494b9e39c939d215b39d413dd97583770887d8 100644 --- a/roles/docker/tasks/networkcreate.yml +++ b/roles/docker/tasks/networkcreate.yml @@ -7,6 +7,5 @@ ipam_config: - subnet: "{{ soctools_network }}" tags: - - start - - startcortex + - start-docker-containers diff --git a/roles/docker/tasks/networkremove.yml b/roles/docker/tasks/networkremove.yml index caec5bc0311dadc3c77a463e33219d7455e1d25c..5a217cea554a724d5d00c87b898b781bd266ddbf 100644 --- a/roles/docker/tasks/networkremove.yml +++ b/roles/docker/tasks/networkremove.yml @@ -5,5 +5,5 @@ name: "{{ soctools_netname }}" state: absent tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/nifi.yml b/roles/docker/tasks/nifi.yml index e85fb61a10993c070180291d6c1ff8d06c74dece..fa32278283ae3aab639460f0e2d010bc700417ab 100644 --- a/roles/docker/tasks/nifi.yml +++ b/roles/docker/tasks/nifi.yml @@ -13,7 +13,7 @@ interactive: "yes" with_items: "{{ groups['nificontainers'] }}" tags: - - start + - start-docker-containers - name: Disconnect Nifi containers from network and remove docker_container: @@ -21,5 +21,5 @@ state: absent with_items: "{{ groups['nificontainers'] }}" tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/odfees.yml b/roles/docker/tasks/odfees.yml index fa35ddd58f54e597b24c030636c34fb650f732b8..7ae76b5de752ab1ff385f49ffc443387f4196dc4 100644 --- a/roles/docker/tasks/odfees.yml +++ b/roles/docker/tasks/odfees.yml @@ -13,7 +13,7 @@ interactive: "yes" with_items: "{{ groups['odfeescontainers'] }}" tags: - - start + - start-docker-containers - name: Disconnect odfe elasticsearch containers from network and remove docker_container: @@ -21,5 +21,5 @@ state: absent with_items: "{{ groups['odfeescontainers'] }}" tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/odfekibana.yml b/roles/docker/tasks/odfekibana.yml index a1c88a2932cd76350e3fedadbec1e2f5511d0748..af5662297eef035ece118686b1ae6c4865f2a082 100644 --- a/roles/docker/tasks/odfekibana.yml +++ b/roles/docker/tasks/odfekibana.yml @@ -13,7 +13,7 @@ interactive: "yes" with_items: "{{ groups['odfekibanacontainers'] }}" tags: - - start + - start-docker-containers - name: Disconnect odfe kibana containers from network and remove docker_container: @@ -21,5 +21,5 @@ state: absent with_items: "{{ groups['odfekibanacontainers'] }}" tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/thehive.yml b/roles/docker/tasks/thehive.yml index 68fdd3c3f21e711ee957feee298b24544e9bbc66..18dc84045eb82814c31ee95dcf49d53db35328e7 100644 --- a/roles/docker/tasks/thehive.yml +++ b/roles/docker/tasks/thehive.yml @@ -8,12 +8,10 @@ networks: - name: "{{ soctools_netname }}" networks_cli_compatible: yes -# published_ports: -# - "9000:9000" interactive: "yes" with_items: "{{ groups['thehive'] }}" tags: - - start + - start-docker-containers - name: Disconnect thehive containers from network and remove docker_container: @@ -21,5 +19,5 @@ state: absent with_items: "{{ groups['thehive'] }}" tags: - - stop + - stop-docker-containers diff --git a/roles/docker/tasks/volumecreate.yml b/roles/docker/tasks/volumecreate.yml index 706652a162e79b0c5defbc6a731fcde9d3decebe..a4e92207fcff83c4d282524f04456c2455b08475 100644 --- a/roles/docker/tasks/volumecreate.yml +++ b/roles/docker/tasks/volumecreate.yml @@ -6,7 +6,7 @@ with_items: - "{{ groups['mysql'] }}" tags: - - start + - start-docker-containers - name: Create cassandra volumes docker_volume: @@ -14,7 +14,7 @@ with_items: - "{{ groups['cassandra'] }}" tags: - - start + - start-docker-containers - name: Create cortex volumes docker_volume: @@ -22,7 +22,7 @@ with_items: - "{{ groups['cortex'] }}" tags: - - start + - start-docker-containers - name: Create NiFi volumes docker_volume: @@ -30,7 +30,7 @@ with_items: - "{{ groups['nificontainers'] }}" tags: - - start + - start-docker-containers - name: Create OpenDistro for Elasticearch volumes docker_volume: @@ -38,5 +38,5 @@ with_items: - "{{ groups['odfeescontainers'] }}" tags: - - start + - start-docker-containers diff --git a/roles/docker/tasks/zookeeper.yml b/roles/docker/tasks/zookeeper.yml index d8cc7f305d94e233a78ae7f9a7e54bbc59081e15..d0b3f71dd99d6d8192e58c560fa817a43757b66f 100644 --- a/roles/docker/tasks/zookeeper.yml +++ b/roles/docker/tasks/zookeeper.yml @@ -11,12 +11,12 @@ env: ALLOW_ANONYMOUS_LOGIN: "yes" tags: - - start + - start-docker-containers - name: Remove zookeeper docker_container: name: "{{ zookeeper_name }}" state: absent tags: - - stop + - stop-docker-containers diff --git a/roles/filebeat/tasks/init.yml b/roles/filebeat/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..67b384685dc3891695436e398be794d5a1e83765 --- /dev/null +++ b/roles/filebeat/tasks/init.yml @@ -0,0 +1,19 @@ +--- +# tasks file for filebeat + +- name: Copy filebeat certificates + copy: + src: "{{ item }}" + dest: "/opt/filebeat/" + mode: 0600 + with_items: + - "{{playbook_dir}}/secrets/CA/issued/filebeat.crt" + - "{{playbook_dir}}/secrets/CA/private/filebeat.key" + become: true + +- name: Copy filebeat configuration file + template: + src: filebeat.yml.j2 + dest: /opt/filebeat/filebeat.yml + become: true + diff --git a/roles/filebeat/tasks/main.yml b/roles/filebeat/tasks/main.yml index 02ac9c7ed8c8f4eba2c4a99295202d1acbd72090..5035f7b8e1f4cab26c1f123921104589c7d56528 100644 --- a/roles/filebeat/tasks/main.yml +++ b/roles/filebeat/tasks/main.yml @@ -1,8 +1,12 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - init - include: stop.yml tags: - stop diff --git a/roles/filebeat/tasks/start.yml b/roles/filebeat/tasks/start.yml index 1944c1f0ec815b4c1075035fc4dc77c2b948f926..f2f3ec495b355b28657f965017148303615e42e4 100644 --- a/roles/filebeat/tasks/start.yml +++ b/roles/filebeat/tasks/start.yml @@ -1,22 +1,6 @@ --- # tasks file for filebeat -- name: Copy filebeat certificates - copy: - src: "{{ item }}" - dest: "/opt/filebeat/" - mode: 0600 - with_items: - - "{{playbook_dir}}/secrets/CA/issued/filebeat.crt" - - "{{playbook_dir}}/secrets/CA/private/filebeat.key" - become: true - -- name: Copy filebeat configuration file - template: - src: filebeat.yml.j2 - dest: /opt/filebeat/filebeat.yml - become: true - - name: Restart filebeat remote_user: root shell: "supervisorctl restart filebeat" diff --git a/roles/haproxy/tasks/init.yml b/roles/haproxy/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..a64cc79d5eb6e090be09a704e4ef1cec57ee0f1d --- /dev/null +++ b/roles/haproxy/tasks/init.yml @@ -0,0 +1,52 @@ +--- +# tasks file for haproxy + +- name: Copy haproxy configuration file + template: + src: haproxy.cfg.j2 + dest: /usr/local/etc/haproxy/haproxy.cfg + +- name: Create required directories + file: + path: "{{ item }}" + state: directory + mode: 0755 + owner: root + group: root + with_items: + - /opt/haproxy + - /etc/ssl/haproxy + +- name: Generate dhparam file for haproxy + shell: "openssl dhparam -out /usr/local/etc/haproxy/dhparam.pem 2048" + +- name: Copy haproxy certificates + copy: + src: "{{ item }}" + dest: "/opt/haproxy/" + mode: 0600 + with_items: + - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" + +- name: Combine crt and key for haproxy + assemble: + src: /opt/haproxy + dest: /etc/ssl/haproxy/{{ inventory_hostname }}.crt + owner: root + mode: 0600 + +- name: Delete temporary files and directory + file: + path: "{{ item }}" + state: absent + with_items: + - /opt/haproxy/{{ inventory_hostname }}.crt + - /opt/haproxy/{{ inventory_hostname }}.key + - /opt/haproxy + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml index 00ff3573b0523cbc4c0f71d1514bc862117d7e6e..d0fb7ab228972bfc8c8469c6df41b91edbef4919 100644 --- a/roles/haproxy/tasks/main.yml +++ b/roles/haproxy/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-haproxy + - init - include: stop.yml tags: - stop diff --git a/roles/haproxy/tasks/start.yml b/roles/haproxy/tasks/start.yml index f33f0eab5622bbca1e9f73dfbe575322aebc9f1b..30ef26f3a1c17bc0a1d3157e0052a7e82012933e 100644 --- a/roles/haproxy/tasks/start.yml +++ b/roles/haproxy/tasks/start.yml @@ -1,55 +1,6 @@ --- # tasks file for haproxy -- name: Copy haproxy configuration file - template: - src: haproxy.cfg.j2 - dest: /usr/local/etc/haproxy/haproxy.cfg - -- name: Create required directories - file: - path: "{{ item }}" - state: directory - mode: 0755 - owner: root - group: root - with_items: - - /opt/haproxy - - /etc/ssl/haproxy - -- name: Generate dhparam file for haproxy - shell: "openssl dhparam -out /usr/local/etc/haproxy/dhparam.pem 2048" - -- name: Copy haproxy certificates - copy: - src: "{{ item }}" - dest: "/opt/haproxy/" - mode: 0600 - with_items: - - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" - -- name: Combine crt and key for haproxy - assemble: - src: /opt/haproxy - dest: /etc/ssl/haproxy/{{ inventory_hostname }}.crt - owner: root - mode: 0600 - -- name: Delete temporary files and directory - file: - path: "{{ item }}" - state: absent - with_items: - - /opt/haproxy/{{ inventory_hostname }}.crt - - /opt/haproxy/{{ inventory_hostname }}.key - - /opt/haproxy - - name: Start haproxy shell: "supervisorctl start haproxy" -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' diff --git a/roles/keycloak/tasks/init.yml b/roles/keycloak/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..8f9bc309114504457d3ee55f03f077fe0e0b08ae --- /dev/null +++ b/roles/keycloak/tasks/init.yml @@ -0,0 +1,103 @@ +--- + +- name: Copy certificates in keycloak x509 conf dir + remote_user: jboss + copy: + src: "{{ item.local }}" + dest: "{{ item.remote }}" + mode: "{{ item.mode}}" + with_items: + - local: "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" + remote: /etc/x509/https/tls.crt + mode: '0644' + - local: "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" + remote: /etc/x509/https/tls.key + mode: '0600' + - local: "{{playbook_dir}}/secrets/CA/ca.crt" + remote: /etc/x509/ca/ca.crt + mode: '0644' + - local: "{{playbook_dir}}/secrets/CA/cacerts.jks" + remote: /opt/jboss/keycloak/cacerts.jks + mode: '0644' + +- name: Generate Keycloak secure config + remote_user: jboss + command: "/opt/jboss/tools/x509.sh" + environment: + X509_CA_BUNDLE: "/etc/x509/ca/ca.crt" + +- name: Set admin password + remote_user: jboss + command: /opt/jboss/keycloak/bin/add-user-keycloak.sh --user "admin" --password "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keykloak_admin')}}" + ignore_errors: True + +- name: Configure logging format + remote_user: jboss + lineinfile: #TODO: Change to community.general.xml + path: /opt/jboss/keycloak/standalone/configuration/standalone.xml + regexp: '.*<formatter name="PATTERN">.*' + line: "<formatter name=\"JSON\"><json-formatter date-format=\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\" pretty-print=\"false\" print-details=\"true\" zone-id=\"UTC\"/></formatter><formatter name=\"PATTERN\">" + +- name: Enable event logging + remote_user: jboss + lineinfile: + path: /opt/jboss/keycloak/standalone/configuration/standalone.xml + regexp: '.*<spi name="eventsStore">.*' + line: '<spi name="eventsListener"><provider name="jboss-logging" enabled="true"><properties><property name="success-level" value="INFO"/><property name="error-level" value="WARN"/></properties></provider></spi><spi name="eventsStore">' + +- name: Specify logging format + remote_user: jboss + lineinfile: + path: /opt/jboss/keycloak/standalone/configuration/standalone.xml + regexp: ".*<named-formatter name=.*" + line: '<named-formatter name="JSON"/>' + +- name: Configure Keycloak start script + remote_user: jboss + template: + src: "{{item}}.j2" + dest: "/opt/jboss/tools/{{item}}" + mode: 0750 + with_items: + - startkeycloak.sh + - initkeycloakrealm.sh + +- name: Start Keycloak IdP + remote_user: root + command: "supervisorctl start keycloak" + +- name: Wait for Keycloak + remote_user: jboss + wait_for: + host: "{{groups['keycloakcontainers'][0]}}" + port: 8443 + state: started + delay: 5 + +- name: Initialize Keycloak realm + remote_user: jboss + command: /opt/jboss/tools/initkeycloakrealm.sh + +- name: Copy secrets from Keycloak + remote_user: jboss + fetch: + src: "{{ item.remote }}" + dest: "{{ item.local }}" + flat: yes + with_items: + - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/nifisecret" + local: "{{playbook_dir}}/secrets/tokens/nifisecret" + - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/kibanasecret" + local: "{{playbook_dir}}/secrets/tokens/kibanasecret" + - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/mispsecret" + local: "{{playbook_dir}}/secrets/tokens/mispsecret" + - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/thehivesecret" + local: "{{playbook_dir}}/secrets/tokens/thehivesecret" + - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/cortexsecret" + local: "{{playbook_dir}}/secrets/tokens/cortexsecret" + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' diff --git a/roles/keycloak/tasks/main.yml b/roles/keycloak/tasks/main.yml index d7668ba48273ed7f841c5e4e868aaa4415817194..7d7bc1619237cef13ed0afe76d63de64e7916ddf 100644 --- a/roles/keycloak/tasks/main.yml +++ b/roles/keycloak/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-keycloak + - init - include: stop.yml tags: - stop diff --git a/roles/keycloak/tasks/start.yml b/roles/keycloak/tasks/start.yml index 8f9bc309114504457d3ee55f03f077fe0e0b08ae..4c8551dcbc0969ff01204128728cc6ba9ea111fc 100644 --- a/roles/keycloak/tasks/start.yml +++ b/roles/keycloak/tasks/start.yml @@ -1,67 +1,5 @@ --- -- name: Copy certificates in keycloak x509 conf dir - remote_user: jboss - copy: - src: "{{ item.local }}" - dest: "{{ item.remote }}" - mode: "{{ item.mode}}" - with_items: - - local: "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" - remote: /etc/x509/https/tls.crt - mode: '0644' - - local: "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" - remote: /etc/x509/https/tls.key - mode: '0600' - - local: "{{playbook_dir}}/secrets/CA/ca.crt" - remote: /etc/x509/ca/ca.crt - mode: '0644' - - local: "{{playbook_dir}}/secrets/CA/cacerts.jks" - remote: /opt/jboss/keycloak/cacerts.jks - mode: '0644' - -- name: Generate Keycloak secure config - remote_user: jboss - command: "/opt/jboss/tools/x509.sh" - environment: - X509_CA_BUNDLE: "/etc/x509/ca/ca.crt" - -- name: Set admin password - remote_user: jboss - command: /opt/jboss/keycloak/bin/add-user-keycloak.sh --user "admin" --password "{{lookup('password', '{{playbook_dir}}/secrets/passwords/keykloak_admin')}}" - ignore_errors: True - -- name: Configure logging format - remote_user: jboss - lineinfile: #TODO: Change to community.general.xml - path: /opt/jboss/keycloak/standalone/configuration/standalone.xml - regexp: '.*<formatter name="PATTERN">.*' - line: "<formatter name=\"JSON\"><json-formatter date-format=\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\" pretty-print=\"false\" print-details=\"true\" zone-id=\"UTC\"/></formatter><formatter name=\"PATTERN\">" - -- name: Enable event logging - remote_user: jboss - lineinfile: - path: /opt/jboss/keycloak/standalone/configuration/standalone.xml - regexp: '.*<spi name="eventsStore">.*' - line: '<spi name="eventsListener"><provider name="jboss-logging" enabled="true"><properties><property name="success-level" value="INFO"/><property name="error-level" value="WARN"/></properties></provider></spi><spi name="eventsStore">' - -- name: Specify logging format - remote_user: jboss - lineinfile: - path: /opt/jboss/keycloak/standalone/configuration/standalone.xml - regexp: ".*<named-formatter name=.*" - line: '<named-formatter name="JSON"/>' - -- name: Configure Keycloak start script - remote_user: jboss - template: - src: "{{item}}.j2" - dest: "/opt/jboss/tools/{{item}}" - mode: 0750 - with_items: - - startkeycloak.sh - - initkeycloakrealm.sh - - name: Start Keycloak IdP remote_user: root command: "supervisorctl start keycloak" @@ -74,30 +12,3 @@ state: started delay: 5 -- name: Initialize Keycloak realm - remote_user: jboss - command: /opt/jboss/tools/initkeycloakrealm.sh - -- name: Copy secrets from Keycloak - remote_user: jboss - fetch: - src: "{{ item.remote }}" - dest: "{{ item.local }}" - flat: yes - with_items: - - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/nifisecret" - local: "{{playbook_dir}}/secrets/tokens/nifisecret" - - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/kibanasecret" - local: "{{playbook_dir}}/secrets/tokens/kibanasecret" - - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/mispsecret" - local: "{{playbook_dir}}/secrets/tokens/mispsecret" - - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/thehivesecret" - local: "{{playbook_dir}}/secrets/tokens/thehivesecret" - - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/cortexsecret" - local: "{{playbook_dir}}/secrets/tokens/cortexsecret" - -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' diff --git a/roles/misp/tasks/init.yml b/roles/misp/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..335ce0b557c855860169aa12e5f55cc92966bc6e --- /dev/null +++ b/roles/misp/tasks/init.yml @@ -0,0 +1,59 @@ +--- + +- name: Set PHP symbolic link + file: + src: /opt/rh/rh-php72/root/bin/php + dest: /usr/bin/php + state: link + +- name: Copy certificates + copy: + src: "{{ item.local }}" + dest: "{{ item.remote }}" + mode: "{{ item.mode}}" + with_items: + - local: "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" + remote: /etc/ssl/certs/misp.crt + mode: '0644' + - local: "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" + remote: /etc/ssl/certs/misp.key + mode: '0600' + - local: "{{playbook_dir}}/secrets/CA/ca.crt" + remote: /etc/ssl/certs/ca.crt + mode: '0644' + - local: "{{playbook_dir}}/secrets/CA/ca.crt" + remote: /etc/pki/ca-trust/source/anchors/ca.crt + mode: '0644' + +- name: Update CA trust + command: update-ca-trust + +- name: Get openid authkey + set_fact: + mispsecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/mispsecret',convert_data=False) | from_json }}" + +- name: Configure Apache web server for misp + template: + src: misp.conf.j2 + dest: /etc/httpd/conf.d/misp.conf + +- name: Configure MISP database access + template: + src: database.php.j2 + dest: /var/www/MISP/app/Config/database.php + +- name: Configure salt + lineinfile: + path: /var/www/MISP/app/Config/config.php + regexp: "'salt'.*=>" + line: "'salt' => '{{lookup('password', '{{playbook_dir}}/secrets/misp_salt')}}'," + +- name: Configure MISP database initialization script + template: + src: checkdb.sh.j2 + dest: /var/www/MISP/checkdb.sh + mode: '0700' + +- name: Check if database is initialized + command: /var/www/MISP/checkdb.sh + diff --git a/roles/misp/tasks/main.yml b/roles/misp/tasks/main.yml index 9b53c6c2b6dd1a2bc3207018e2524abfc390af9f..73e30bce11ebbb36a6458d7a1b22649d07d686ae 100644 --- a/roles/misp/tasks/main.yml +++ b/roles/misp/tasks/main.yml @@ -1,14 +1,21 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-misp + - init - include: config.yml tags: - - start + - init - config - include: stop.yml - tags: stop + tags: + - stop + - stop-misp - include: update-config.yml tags: - update-config diff --git a/roles/misp/tasks/start.yml b/roles/misp/tasks/start.yml index c07917fd83fe41509adf423109b20bfc9e781744..10e9b7915a7a692070839ee143689927170364d0 100644 --- a/roles/misp/tasks/start.yml +++ b/roles/misp/tasks/start.yml @@ -1,62 +1,5 @@ --- -- name: Set PHP symbolic link - file: - src: /opt/rh/rh-php72/root/bin/php - dest: /usr/bin/php - state: link - -- name: Copy certificates - copy: - src: "{{ item.local }}" - dest: "{{ item.remote }}" - mode: "{{ item.mode}}" - with_items: - - local: "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" - remote: /etc/ssl/certs/misp.crt - mode: '0644' - - local: "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" - remote: /etc/ssl/certs/misp.key - mode: '0600' - - local: "{{playbook_dir}}/secrets/CA/ca.crt" - remote: /etc/ssl/certs/ca.crt - mode: '0644' - - local: "{{playbook_dir}}/secrets/CA/ca.crt" - remote: /etc/pki/ca-trust/source/anchors/ca.crt - mode: '0644' - -- name: Update CA trust - command: update-ca-trust - -- name: Get openid authkey - set_fact: - mispsecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/mispsecret',convert_data=False) | from_json }}" - -- name: Configure Apache web server for misp - template: - src: misp.conf.j2 - dest: /etc/httpd/conf.d/misp.conf - -- name: Configure MISP database access - template: - src: database.php.j2 - dest: /var/www/MISP/app/Config/database.php - -- name: Configure salt - lineinfile: - path: /var/www/MISP/app/Config/config.php - regexp: "'salt'.*=>" - line: "'salt' => '{{lookup('password', '{{playbook_dir}}/secrets/misp_salt')}}'," - -- name: Configure MISP database initialization script - template: - src: checkdb.sh.j2 - dest: /var/www/MISP/checkdb.sh - mode: '0700' - -- name: Check if database is initialized - command: /var/www/MISP/checkdb.sh - - name: Start php-fpm command: "supervisorctl start php-fpm" diff --git a/roles/misp/tasks/stop.yml b/roles/misp/tasks/stop.yml index 6a56d87d74255095f17ca8ac2db562596446f586..769afaeb35cb773fc6b86af6a26fd7f58df92ddd 100644 --- a/roles/misp/tasks/stop.yml +++ b/roles/misp/tasks/stop.yml @@ -12,6 +12,5 @@ command: "supervisorctl stop misp-modules" - name: Stop workers - #command: "supervisorctl start workers" - command: 'su - -s /bin/bash -c "/var/www/MISP/app/Console/worker/stop.sh" apache' + command: "supervisorctl stop workers" diff --git a/roles/mysql/tasks/init.yml b/roles/mysql/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..e360aa8ddafa8c20571b2e75e7e18540a5b860e9 --- /dev/null +++ b/roles/mysql/tasks/init.yml @@ -0,0 +1,7 @@ +--- + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' diff --git a/roles/mysql/tasks/main.yml b/roles/mysql/tasks/main.yml index 69395fe3e0562adca1f273db7d17f8e3b1451ee6..7b6d8eeb58e95d3637b229da518e2175e3458569 100644 --- a/roles/mysql/tasks/main.yml +++ b/roles/mysql/tasks/main.yml @@ -2,10 +2,17 @@ - include: secure.yml tags: - - start + - init - include: misp.yml + tags: + - init +- include: init.yml + tags: + - init +- include: start.yml tags: - start + - start-mysql - include: stop.yml tags: - stop diff --git a/roles/mysql/tasks/misp.yml b/roles/mysql/tasks/misp.yml index c541e99243876ad9739ec4b52bc946b481ef6274..471da9d601fc8e2429be34902180778e6cbefbd3 100644 --- a/roles/mysql/tasks/misp.yml +++ b/roles/mysql/tasks/misp.yml @@ -14,16 +14,3 @@ priv: "{{misp_dbname}}.*:ALL" with_items: "{{groups['mispcontainers']}}" -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' - - -# CREATE DATABASE IF NOT EXISTS {{misp_dbname}}; -# {% for misp_host in groups['mispcontainers'] %} -# GRANT USAGE on *.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}' IDENTIFIED by '{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}}'; -# GRANT ALL PRIVILEGES on {{misp_dbname}}.* to '{{misp_dbuser}}'@'{{misp_host}}.{{soctools_netname}}'; -# {% endfor %} -# diff --git a/roles/mysql/tasks/secure.yml b/roles/mysql/tasks/secure.yml index 80db96cf207efd2ba5b4ec0c5aa4828041a6e980..23dccd4a0b18a7fae8dfdfff2d32e465301ba663 100644 --- a/roles/mysql/tasks/secure.yml +++ b/roles/mysql/tasks/secure.yml @@ -5,8 +5,6 @@ name: root host_all: yes password: "{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}" - tags: - - start ignore_errors: true - name: Configure root access to db @@ -14,59 +12,15 @@ src: dotmy.cnf.j2 dest: /root/.my.cnf mode: 0600 - tags: - - start - name: Remove all anonymous user accounts mysql_user: name: '' host_all: yes state: absent - tags: - - start - name: Remove MySQL test database mysql_db: name: test state: absent - tags: - - start - -# template: -# src: mysql_secure.sql.j2 -# dest: /opt/rh/rh-mariadb103/root/root/mysql_secure.sql -# tags: -# - start -# - reconf -# -# - name: Apply mysql conf -# command: "/opt/rh/rh-mariadb103/root/usr/bin/mysql -v -v -v -u root -e 'source /opt/rh/rh-mariadb103/root/root/mysql_secure.sql'" -# register: mysqlconf -# tags: -# - start -# - reconf -# -# - name: show output from mysql command -# debug: msg="{{mysqlconf.stdout}}" -# tags: -# - start -# -# -# -# UPDATE mysql.user SET Password=PASSWORD('{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_root')}}') WHERE User='root'; -# DELETE FROM mysql.user WHERE User=''; -# DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1'); -# DROP DATABASE IF EXISTS test; -# DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%'; -# -# -# -# -# #!/bin/bash -x -# MISPINIT=$(echo "SELECT COUNT(DISTINCT 'table_name') FROM information_schema.columns WHERE table_schema = '{{misp_dbname}}';" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}}) -# #MISPINIT=$(echo "select count(id) from users;" | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}}) -# if [ ${MISPINIT} == "0" ]; then -# cat /var/www/MISP/INSTALL/MYSQL.sql | mysql -s -h {{groups['mysql'][0]}} -u {{misp_dbuser}} -p{{lookup('password', '{{playbook_dir}}/secrets/passwords/mysql_misp')}} {{misp_dbname}} -# touch /var/www/MISP/dbchecked-$(date +%Y%m%d_%H%M%S) -# fi diff --git a/roles/mysql/tasks/start.yml b/roles/mysql/tasks/start.yml new file mode 100644 index 0000000000000000000000000000000000000000..e072ee218df7a8bf49e317b0f76346aaa6f726f2 --- /dev/null +++ b/roles/mysql/tasks/start.yml @@ -0,0 +1,4 @@ +--- + +- name: Start MySQL + shell: "supervisorctl start mysql" diff --git a/roles/nifi/tasks/init.yml b/roles/nifi/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..58b26f3fc2507c71a56dde9ac0709feda5c4e6d4 --- /dev/null +++ b/roles/nifi/tasks/init.yml @@ -0,0 +1,185 @@ +--- + +- name: Copy cacert to ca-trust dir + remote_user: root + copy: + src: "{{playbook_dir}}/secrets/CA/ca.crt" + dest: /etc/pki/ca-trust/source/anchors/ca.crt + +- name: Install cacert to root truststore + remote_user: root + command: "update-ca-trust" + +- name: Copy certificates in NiFi conf dir + remote_user: nifi + copy: + src: "{{ item }}" + dest: "conf/" + with_items: + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" + - "{{playbook_dir}}/secrets/CA/cacerts.jks" + - common-cacerts.jks + +- name: Check if flow.xml already exists + remote_user: nifi + stat: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + register: flowfile + +- name: Configure flow.xml + remote_user: nifi + template: + src: "flow.xml.j2" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + when: not flowfile.stat.exists + +- name: Gzip flow.xml + remote_user: nifi + archive: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" + format: gz + when: not flowfile.stat.exists + +- name: Get openid authkey + remote_user: nifi + set_fact: + nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}" + +- name: Configure NiFi boostrap properties + remote_user: nifi + template: + src: bootstrap.conf.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf" + +- name: Configure NiFi properties for secure servers + remote_user: nifi + template: + src: nifi.properties.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties" + +- name: Copy authorizations.xml + remote_user: nifi + copy: + src: "authorizations.xml" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml" + +- name: Configure users + remote_user: nifi + template: + src: users.xml.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml" + +- name: Configure NiFi authorizers for secure servers + remote_user: nifi + template: + src: authorizers.xml.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml" + +- name: Create conf/enrich dir + remote_user: nifi + file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory + +- name: Create conf/enrich/freq dir + remote_user: nifi + file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory + +- name: Download freq processor + remote_user: nifi + get_url: + url: 'https://gitlab.geant.org/gn4-3-wp8-t3.1-soc/nifi-processors/-/raw/master/scripts/freq/{{ item }}' + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/" + with_items: + - alexa.json + - freq.py + - freqProcessor.py + +- name: Copy empty GeoLite2-City database + remote_user: nifi + copy: + src: GeoLite2-City.mmdb + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb" + +- name: Copy CountriesWithRegionalCodes.csv + remote_user: nifi + copy: + src: CountriesWithRegionalCodes.csv + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv" + +- name: Copy grok libraries + copy: + src: haproxy.groklib + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib" + +- name: Create empty list of Tor nodes + remote_user: nifi + copy: + content: "ip_addr,value" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv" + force: no + +- name: Download umbrella-top-1m.csv.zip + remote_user: nifi + local_action: + module: get_url + url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip + dest: "/tmp/umbrella-top-1m.csv.zip" + run_once: True + +- name: Unzip umbrella-top-1m.csv.zip + remote_user: nifi + local_action: + module: unarchive + src: "/tmp/umbrella-top-1m.csv.zip" + dest: "/tmp" + run_once: True + +- name: Copy umbrella-top-1m.csv + remote_user: nifi + copy: + src: "/tmp/top-1m.csv" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" + +- name: Add header to umbrella-top-1m.csv + remote_user: nifi + lineinfile: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" + line: 'index,domain' + insertbefore: BOF + +- name: Download alexa-top-1m.csv.zip + remote_user: nifi + local_action: + module: get_url + url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip + dest: "/tmp/alexa-top-1m.csv.zip" + run_once: True + +- name: Unzip alexa-top-1m.csv.zip + remote_user: nifi + local_action: + module: unarchive + src: "/tmp/alexa-top-1m.csv.zip" + dest: "/tmp" + run_once: True + +- name: Copy alexa-top-1m.csv + remote_user: nifi + copy: + src: "/tmp/top-1m.csv" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" + +- name: Add header to alexa-top-1m.csv + remote_user: nifi + lineinfile: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" + line: 'index,domain' + insertbefore: BOF + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + + diff --git a/roles/nifi/tasks/main.yml b/roles/nifi/tasks/main.yml index d4ab3ad3a89413bf3d6a848822562a5ccb6d92ca..5667605a7cc6a79852dc33bb984b607f2675123d 100644 --- a/roles/nifi/tasks/main.yml +++ b/roles/nifi/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-nifi + - init - include: stop.yml tags: - stop diff --git a/roles/nifi/tasks/start.yml b/roles/nifi/tasks/start.yml index ecccbbd86f42819230a8f25840d036ee07991857..31ee2ec21369399bc758276ea0db8e37a81eb648 100644 --- a/roles/nifi/tasks/start.yml +++ b/roles/nifi/tasks/start.yml @@ -1,204 +1,6 @@ --- -- name: Copy cacert to ca-trust dir - remote_user: root - copy: - src: "{{playbook_dir}}/secrets/CA/ca.crt" - dest: /etc/pki/ca-trust/source/anchors/ca.crt - -- name: Install cacert to root truststore - remote_user: root - command: "update-ca-trust" - -- name: Copy certificates in NiFi conf dir - remote_user: nifi - copy: - src: "{{ item }}" - dest: "conf/" - with_items: - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" - - "{{playbook_dir}}/secrets/CA/cacerts.jks" - - common-cacerts.jks - -- name: Check if flow.xml already exists - remote_user: nifi - stat: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" - register: flowfile - -- name: Configure flow.xml - remote_user: nifi - template: - src: "flow.xml.j2" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" - when: not flowfile.stat.exists - -- name: Gzip flow.xml - remote_user: nifi - archive: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" - format: gz - when: not flowfile.stat.exists - -- name: Get openid authkey - remote_user: nifi - set_fact: - nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}" - -- name: Configure NiFi boostrap properties - remote_user: nifi - template: - src: bootstrap.conf.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf" - -- name: Configure NiFi properties for secure servers - remote_user: nifi - template: - src: nifi.properties.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties" - -- name: Copy authorizations.xml - remote_user: nifi - copy: - src: "authorizations.xml" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml" - -- name: Configure users - remote_user: nifi - template: - src: users.xml.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml" - -- name: Configure NiFi authorizers for secure servers - remote_user: nifi - template: - src: authorizers.xml.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml" - -- name: Create conf/enrich dir - remote_user: nifi - file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory - -- name: Create conf/enrich/freq dir - remote_user: nifi - file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory - -- name: Download freq processor - remote_user: nifi - get_url: - url: 'https://gitlab.geant.org/gn4-3-wp8-t3.1-soc/nifi-processors/-/raw/master/scripts/freq/{{ item }}' - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/" - with_items: - - alexa.json - - freq.py - - freqProcessor.py - -- name: Copy empty GeoLite2-City database - remote_user: nifi - copy: - src: GeoLite2-City.mmdb - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb" - -- name: Copy CountriesWithRegionalCodes.csv - remote_user: nifi - copy: - src: CountriesWithRegionalCodes.csv - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv" - -- name: Copy grok libraries - copy: - src: haproxy.groklib - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib" - -- name: Create empty list of Tor nodes - remote_user: nifi - copy: - content: "ip_addr,value" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv" - force: no - -- name: Download umbrella-top-1m.csv.zip - remote_user: nifi - local_action: - module: get_url - url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip - dest: "/tmp/umbrella-top-1m.csv.zip" - run_once: True - -- name: Unzip umbrella-top-1m.csv.zip - remote_user: nifi - local_action: - module: unarchive - src: "/tmp/umbrella-top-1m.csv.zip" - dest: "/tmp" - run_once: True - -- name: Copy umbrella-top-1m.csv - remote_user: nifi - copy: - src: "/tmp/top-1m.csv" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" - -- name: Add header to umbrella-top-1m.csv - remote_user: nifi - lineinfile: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" - line: 'index,domain' - insertbefore: BOF - -- name: Download alexa-top-1m.csv.zip - remote_user: nifi - local_action: - module: get_url - url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip - dest: "/tmp/alexa-top-1m.csv.zip" - run_once: True - -- name: Unzip alexa-top-1m.csv.zip - remote_user: nifi - local_action: - module: unarchive - src: "/tmp/alexa-top-1m.csv.zip" - dest: "/tmp" - run_once: True - -- name: Copy alexa-top-1m.csv - remote_user: nifi - copy: - src: "/tmp/top-1m.csv" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" - -- name: Add header to alexa-top-1m.csv - remote_user: nifi - lineinfile: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" - line: 'index,domain' - insertbefore: BOF - - name: Start NiFi remote_user: root command: "supervisorctl start nifi" -#- name: check reachable hosts -# gather_facts: no -# tasks: -# - command: ping -c1 {{ inventory_hostname }} -# delegate_to: localhost -# register: ping_result -# ignore_errors: yes -# - group_by: key=reachable -# when: ping_result|success - -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' - -#- name: Copy flow from NiFi -# fetch: -# src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" -# dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz" -# flat: yes - diff --git a/roles/odfees/tasks/init.yml b/roles/odfees/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..0eaedc5b5529f0568b706fea12f87d9d64c0489f --- /dev/null +++ b/roles/odfees/tasks/init.yml @@ -0,0 +1,100 @@ +--- + +- name: Copy cacert to ca-trust dir + remote_user: root + copy: + src: "{{playbook_dir}}/secrets/CA/ca.crt" + dest: /etc/pki/ca-trust/source/anchors/ca.crt + +- name: Install cacert to root truststore + remote_user: root + command: "update-ca-trust" + +- name: Copy certificates in odfe conf dir + remote_user: elasticsearch + copy: + src: "{{ item }}" + dest: "config/" + mode: 0600 + with_items: + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" + - "{{playbook_dir}}/secrets/CA/cacerts.jks" + - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12" + +- name: Configure sysconfig + remote_user: elasticsearch + template: + src: sysconfig_elasticsearch.j2 + dest: sysconfig_elasticsearch + +- name: Copy sysconfig to /etc + remote_user: elasticsearch + command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch" + +- name: Configure odfe properties + remote_user: elasticsearch + template: + src: "config/{{item}}.j2" + dest: "config/{{item}}" + with_items: + - elasticsearch.yml + - jvm.options + - log4j2.properties + +- name: Change password for admin + remote_user: elasticsearch + command: "bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}" + register: adminhash + # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname" + +- set_fact: + adminhashpwd: "{{ adminhash.stdout }}" + #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}" + remote_user: elasticsearch + +- name: Change password for cortex + remote_user: elasticsearch + command: "bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_odfe')}}" + register: cortexhash + # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname" + +- set_fact: + cortexhashpwd: "{{ cortexhash.stdout }}" + #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}" + remote_user: elasticsearch + +- name: Configure opendistro_security properties + remote_user: elasticsearch + template: + src: "securityconfig/{{item}}.j2" + dest: "plugins/opendistro_security/securityconfig/{{item}}" + with_items: + - internal_users.yml + - config.yml + - roles_mapping.yml + + +- name: Start OpenDistro for Elasticsearch + remote_user: root + command: "supervisorctl start odfe" + +- name: Wait for ElasticSearch + remote_user: root + wait_for: + host: "{{groups['odfeescontainers'][0]}}" + port: 9200 + state: started + delay: 5 + +- name: Configure OpenDistro security + remote_user: elasticsearch + command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster" + when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname" + +- name: Set Autostart for supervisord's services + remote_user: root + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + diff --git a/roles/odfees/tasks/main.yml b/roles/odfees/tasks/main.yml index 016e8786ed6a31238dd2c7efc68e9aec7c7a9912..2e7a390abbb8ce24c2b29629f37d448f87530a9d 100644 --- a/roles/odfees/tasks/main.yml +++ b/roles/odfees/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-odfees + - init - include: stop.yml tags: - stop diff --git a/roles/odfees/tasks/start.yml b/roles/odfees/tasks/start.yml index 622a1cd65e9e634472048a5ab79207ffa60850fd..953b53fdb88a6c8044581d374c3ba7620969faf8 100644 --- a/roles/odfees/tasks/start.yml +++ b/roles/odfees/tasks/start.yml @@ -1,81 +1,5 @@ --- -- name: Copy cacert to ca-trust dir - remote_user: root - copy: - src: "{{playbook_dir}}/secrets/CA/ca.crt" - dest: /etc/pki/ca-trust/source/anchors/ca.crt - -- name: Install cacert to root truststore - remote_user: root - command: "update-ca-trust" - -- name: Copy certificates in odfe conf dir - remote_user: elasticsearch - copy: - src: "{{ item }}" - dest: "config/" - mode: 0600 - with_items: - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" - - "{{playbook_dir}}/secrets/CA/cacerts.jks" - - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12" - -- name: Configure sysconfig - remote_user: elasticsearch - template: - src: sysconfig_elasticsearch.j2 - dest: sysconfig_elasticsearch - -- name: Copy sysconfig to /etc - remote_user: elasticsearch - command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch" - -- name: Configure odfe properties - remote_user: elasticsearch - template: - src: "config/{{item}}.j2" - dest: "config/{{item}}" - with_items: - - elasticsearch.yml - - jvm.options - - log4j2.properties - -- name: Change password for admin - remote_user: elasticsearch - command: "bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/odfees_adminpass')}}" - register: adminhash - # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname" - -- set_fact: - adminhashpwd: "{{ adminhash.stdout }}" - #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}" - remote_user: elasticsearch - -- name: Change password for cortex - remote_user: elasticsearch - command: "bash plugins/opendistro_security/tools/hash.sh -p {{lookup('password', '{{playbook_dir}}/secrets/passwords/cortex_odfe')}}" - register: cortexhash - # when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname" - -- set_fact: - cortexhashpwd: "{{ cortexhash.stdout }}" - #adminhashpwd: "{{ hostvars[groups['odfeescontainers'][0]]['adminhash.stdout'] }}" - remote_user: elasticsearch - -- name: Configure opendistro_security properties - remote_user: elasticsearch - template: - src: "securityconfig/{{item}}.j2" - dest: "plugins/opendistro_security/securityconfig/{{item}}" - with_items: - - internal_users.yml - - config.yml - - roles_mapping.yml - -#- name: Exit here to test ODFE -# meta: end_play - - name: Start OpenDistro for Elasticsearch remote_user: root command: "supervisorctl start odfe" @@ -88,15 +12,3 @@ state: started delay: 5 -- name: Configure OpenDistro security - remote_user: elasticsearch - command: "bash ./plugins/opendistro_security/tools/securityadmin.sh -h {{groups['odfeescontainers'][0]}} -cd /usr/share/elasticsearch/plugins/opendistro_security/securityconfig/ -ks '/usr/share/elasticsearch/config/{{soctools_users[0].CN}}.p12' -kspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} {{lookup('password','{{playbook_dir}}/secrets/passwords/{{soctools_users[0].CN}}')}} -ts /usr/share/elasticsearch/config/cacerts.jks -tspass {{lookup('password', '{{playbook_dir}}/secrets/passwords/truststore')}} -cn soctools-cluster" - when: "'{{groups['odfeescontainers'][0]}}' in inventory_hostname" - -- name: Set Autostart for supervisord's services - remote_user: root - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' - diff --git a/roles/odfekibana/tasks/init.yml b/roles/odfekibana/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..a6b8b05a2dfe1c662f73f4f3bdd3bb2737d88ff4 --- /dev/null +++ b/roles/odfekibana/tasks/init.yml @@ -0,0 +1,114 @@ +--- + +- name: Copy cacert to ca-trust dir + remote_user: root + copy: + src: "{{playbook_dir}}/secrets/CA/ca.crt" + dest: /etc/pki/ca-trust/source/anchors/ca.crt + +- name: Install cacert to root truststore + remote_user: root + command: "update-ca-trust" + +- name: Copy certificates in odfe kibana conf dir + remote_user: kibana + copy: + src: "{{ item }}" + dest: "config/" + mode: 0600 + with_items: + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" + - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" + - "{{playbook_dir}}/secrets/CA/cacerts.jks" + - "{{playbook_dir}}/secrets/CA/ca.crt" + - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12" + +- name: Get openid authkey + remote_user: kibana + set_fact: + kibanasecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/kibanasecret',convert_data=False) | from_json }}" + +- name: Configure odfe kibana properties + remote_user: kibana + template: + src: "{{item}}.j2" + dest: "config/{{item}}" + with_items: + - kibana.yml + +- name: Configure odfe kibana start script + remote_user: kibana + template: + src: "{{item}}.j2" + dest: "{{item}}" + mode: 0750 + with_items: + - startkibana.sh + +- name: Generate configuration for thehive_button plugin + remote_user: kibana + template: + src: files/env.js.j2 + dest: "/usr/share/kibana/plugins/thehive_button/public/env.js" + owner: kibana + group: kibana + + +- name: Start Kibana + remote_user: root + shell: "supervisorctl start kibana" + +- name: Wait for Kibana + remote_user: kibana + wait_for: + host: "{{groups['odfekibanacontainers'][0]}}" + port: 5601 + state: started + delay: 5 + +- name: Check Kibana health + remote_user: kibana + shell: 'curl -k -b /tmp/cookie.txt -c /tmp/cookie.txt -X "GET" "https://{{soctoolsproxy}}:5601/api/status" \ + | egrep status....overall....state...green' + register: result + until: result.rc == 0 + retries: 90 + delay: 2 + ignore_errors: yes + +- name: Copy kibana_graphs.ndjson to container + remote_user: kibana + template: + src: "kibana_graphs.ndjson.j2" + dest: /tmp/kibana_graphs.ndjson + +- name: Import graphs to kibana + remote_user: kibana + shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/saved_objects/_import?overwrite=true" \ + -b /tmp/cookie.txt -c /tmp/cookie.txt \ + -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \ + -H "kbn-xsrf: reporting" -H "Content-Type: multipart/form-data" \ + -F "file=@/tmp/kibana_graphs.ndjson"' + ignore_errors: True + +- name: Copy role modification json to container + remote_user: kibana + template: + src: "role.json.j2" + dest: /tmp/role.json + +- name: Grant admin permissions to users + remote_user: kibana + shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/v1/configuration/rolesmapping/all_access" \ + -b /tmp/cookie.txt -c /tmp/cookie.txt \ + -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \ + -H "kbn-xsrf: reporting" -H "Content-Type: application/json" \ + -d @/tmp/role.json' + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + diff --git a/roles/odfekibana/tasks/main.yml b/roles/odfekibana/tasks/main.yml index 1662efd600ad1a43e47bc3c40acb09fe69599dda..429925b89da09f01c8de81db6edcdbeb6be5d407 100644 --- a/roles/odfekibana/tasks/main.yml +++ b/roles/odfekibana/tasks/main.yml @@ -1,8 +1,13 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-odfekibana + - init - include: stop.yml tags: - stop diff --git a/roles/odfekibana/tasks/restart.yml b/roles/odfekibana/tasks/restart.yml index 7ef339035ddb769ec70ffad981b8a36959f735e1..cadeb7e282e7d8718218791bdb6c81869810006b 100644 --- a/roles/odfekibana/tasks/restart.yml +++ b/roles/odfekibana/tasks/restart.yml @@ -22,11 +22,3 @@ delay: 2 ignore_errors: yes -- name: Import graphs to kibana - remote_user: kibana - shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/saved_objects/_import?overwrite=true" \ - -b /tmp/cookie.txt -c /tmp/cookie.txt \ - -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \ - -H "kbn-xsrf: reporting" -H "Content-Type: multipart/form-data" \ - -F "file=@/tmp/kibana_graphs.ndjson"' - diff --git a/roles/odfekibana/tasks/start.yml b/roles/odfekibana/tasks/start.yml index a067386d233ebcbd229b9297436097595dc2346f..b22cd0de212ad38926b46e7c1d845f8581b9147a 100644 --- a/roles/odfekibana/tasks/start.yml +++ b/roles/odfekibana/tasks/start.yml @@ -1,83 +1,5 @@ --- -#- name: Create config directory -# file: -# name: config -# state: directory -# mode: 0700 - -- name: Copy cacert to ca-trust dir - remote_user: root - copy: - src: "{{playbook_dir}}/secrets/CA/ca.crt" - dest: /etc/pki/ca-trust/source/anchors/ca.crt - -- name: Install cacert to root truststore - remote_user: root - command: "update-ca-trust" - -- name: Copy certificates in odfe kibana conf dir - remote_user: kibana - copy: - src: "{{ item }}" - dest: "config/" - mode: 0600 - with_items: - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" - - "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" - - "{{playbook_dir}}/secrets/CA/cacerts.jks" - - "{{playbook_dir}}/secrets/CA/ca.crt" - - "{{playbook_dir}}/secrets/CA/private/{{soctools_users[0].CN}}.p12" - -- name: Get openid authkey - remote_user: kibana - set_fact: - kibanasecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/kibanasecret',convert_data=False) | from_json }}" - -#- name: Configure sysconfig -# template: -# src: sysconfig_elasticsearch.j2 -# dest: sysconfig_elasticsearch -# -#- name: Copy sysconfig to /etc -# command: "cp sysconfig_elasticsearch /etc/sysconfig/elasticsearch" - -# lineinfile: -# path: /etc/sysconfig/elasticsearch -# regexp: '^ES_PATH_CONF=' -# line: ES_PATH_CONF=/usr/share/elasticsearch/config - -- name: Configure odfe kibana properties - remote_user: kibana - template: - src: "{{item}}.j2" - dest: "config/{{item}}" - with_items: - - kibana.yml - -- name: Configure odfe kibana start script - remote_user: kibana - template: - src: "{{item}}.j2" - dest: "{{item}}" - mode: 0750 - with_items: - - startkibana.sh - -#- name: Exit here to test ODFE -# meta: end_play - - -- name: Generate configuration for thehive_button plugin - remote_user: kibana - template: - src: files/env.js.j2 - dest: "/usr/share/kibana/plugins/thehive_button/public/env.js" - owner: kibana - group: kibana - - - name: Start Kibana remote_user: root shell: "supervisorctl start kibana" @@ -100,68 +22,3 @@ delay: 2 ignore_errors: yes -#- name: Copy tenant.json to container -# remote_user: kibana -# copy: -# src: "files/tenant.json" -# dest: /tmp/tenant.json -# -#- name: change tenant to global -# shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/v1/multitenancy/tenant" \ -# -b /tmp/cookie.txt -c /tmp/cookie.txt \ -# -k --user admin:{{ odfees_adminpass }} \ -# -H "kbn-xsrf: reporting" -H "Content-Type: application/json" \ -# -d @/tmp/tenant.json' - -- name: Copy kibana_graphs.ndjson to container - remote_user: kibana - template: - src: "kibana_graphs.ndjson.j2" - dest: /tmp/kibana_graphs.ndjson - -- name: Import graphs to kibana - remote_user: kibana - shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/saved_objects/_import?overwrite=true" \ - -b /tmp/cookie.txt -c /tmp/cookie.txt \ - -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \ - -H "kbn-xsrf: reporting" -H "Content-Type: multipart/form-data" \ - -F "file=@/tmp/kibana_graphs.ndjson"' - ignore_errors: True - -- name: Copy role modification json to container - remote_user: kibana - template: - src: "role.json.j2" - dest: /tmp/role.json - -- name: Grant admin permissions to users - remote_user: kibana - shell: 'curl -X "POST" "https://{{soctoolsproxy}}:5601/api/v1/configuration/rolesmapping/all_access" \ - -b /tmp/cookie.txt -c /tmp/cookie.txt \ - -k --user admin:{{lookup("password", "{{playbook_dir}}/secrets/passwords/odfees_adminpass")}} \ - -H "kbn-xsrf: reporting" -H "Content-Type: application/json" \ - -d @/tmp/role.json' - -#- name: cleanup temporary files for kibana_graph import -# shell: '/bin/rm -rf /tmp/cookie.txt /tmp/kibana_graphs.ndjson /tmp/tenant.json' -# ignore_errors: true - -#- name: check reachable hosts -# gather_facts: no -# tasks: -# - command: ping -c1 {{ inventory_hostname }} -# delegate_to: localhost -# register: ping_result -# ignore_errors: yes -# - group_by: key=reachable -# when: ping_result|success - -#- name: Stop OpenDistro Kibana for Elasticsearch -# command: "pkill -SIGTERM -F {{inventory_hostname}}.pid" - -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' - diff --git a/roles/thehive/tasks/init.yml b/roles/thehive/tasks/init.yml new file mode 100644 index 0000000000000000000000000000000000000000..97993c88114e33ee306fa8ffa001952433ce1044 --- /dev/null +++ b/roles/thehive/tasks/init.yml @@ -0,0 +1,49 @@ +--- + +- name: Copy cacert to ca-trust dir + remote_user: root + copy: + src: "{{playbook_dir}}/secrets/CA/ca.crt" + dest: /etc/pki/ca-trust/source/anchors/ca.crt + +- name: Install cacert to root truststore + remote_user: root + command: "update-ca-trust" + +- name: Copy certificate in thehive conf dir + copy: + src: "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" + dest: "/etc/thehive/{{ inventory_hostname }}.crt" + mode: 0600 + +- name: Copy certificate key in thehive conf dir + copy: + src: "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" + dest: "/etc/thehive/{{ inventory_hostname }}.key" + mode: 0600 + +- name: Copy CA certificates in thehive conf dir + copy: + src: "{{playbook_dir}}/secrets/CA/{{ item }}" + dest: "/etc/thehive/{{ item }}" + mode: 0600 + with_items: + - "cacerts.jks" + - "ca.crt" + +- name: Get openid authkey + set_fact: + thehivesecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/thehivesecret',convert_data=False) | from_json}}" + +- name: Configure TheHive + remote_user: thehive + template: + src: application.conf.j2 + dest: /etc/thehive/application.conf + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + diff --git a/roles/thehive/tasks/main.yml b/roles/thehive/tasks/main.yml index def4e3dee132cb841acaf437380a250cd9e0b786..cb0d5d73c654e36ad22216d99a47d3226b74ac79 100644 --- a/roles/thehive/tasks/main.yml +++ b/roles/thehive/tasks/main.yml @@ -1,20 +1,25 @@ --- +- include: init.yml + tags: + - init - include: start.yml tags: - start + - start-thehive + - init - include: adminuser.yml tags: - - start + - init - include: organization.yml tags: - - start + - init - include: kibanauser.yml tags: - - start + - init - include: createusers.yml tags: - - start + - init - create-thehive-users - createusers - include: stop.yml diff --git a/roles/thehive/tasks/start.yml b/roles/thehive/tasks/start.yml index 9521b5752fe9c257a05e571980fa38ae2b0c599d..f5f467baaae326ae3fb21dc1ece495683ef0d4f2 100644 --- a/roles/thehive/tasks/start.yml +++ b/roles/thehive/tasks/start.yml @@ -1,46 +1,5 @@ --- -- name: Copy cacert to ca-trust dir - remote_user: root - copy: - src: "{{playbook_dir}}/secrets/CA/ca.crt" - dest: /etc/pki/ca-trust/source/anchors/ca.crt - -- name: Install cacert to root truststore - remote_user: root - command: "update-ca-trust" - -- name: Copy certificate in thehive conf dir - copy: - src: "{{playbook_dir}}/secrets/CA/issued/{{ inventory_hostname }}.crt" - dest: "/etc/thehive/{{ inventory_hostname }}.crt" - mode: 0600 - -- name: Copy certificate key in thehive conf dir - copy: - src: "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.key" - dest: "/etc/thehive/{{ inventory_hostname }}.key" - mode: 0600 - -- name: Copy CA certificates in thehive conf dir - copy: - src: "{{playbook_dir}}/secrets/CA/{{ item }}" - dest: "/etc/thehive/{{ item }}" - mode: 0600 - with_items: - - "cacerts.jks" - - "ca.crt" - -- name: Get openid authkey - set_fact: - thehivesecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/thehivesecret',convert_data=False) | from_json}}" - -- name: Configure TheHive - remote_user: thehive - template: - src: application.conf.j2 - dest: /etc/thehive/application.conf - - name: Start TheHive remote_user: root command: "supervisorctl start thehive" @@ -53,9 +12,4 @@ state: started delay: 5 -- name: Set Autostart for supervisord's services - replace: - path: /etc/supervisord.conf - regexp: '^autostart=false$' - replace: 'autostart=true' diff --git a/soctools.yml b/soctools.yml index 4e7f89ba1912be4eff657c45c7a47ed450946a6c..2dd3142cfc95b19f689cea9ef193843e584c128c 100644 --- a/soctools.yml +++ b/soctools.yml @@ -1,8 +1,16 @@ --- +- name: Manage Docker Containers + import_playbook: dockercontainers.yml + when: "'start-docker-containers' in ansible_run_tags or 'stop-docker-containers' in ansible_run_tags" + +- name: Init and start soctools cluster + import_playbook: initsoctools.yml + when: "'init' in ansible_run_tags or 'config' in ansible_run_tags" + - name: Start soctools cluster import_playbook: startsoctools.yml - when: "'start' in ansible_run_tags or 'config' in ansible_run_tags" + when: "'start' in ansible_run_tags or 'config' in ansible_run_tags or 'start-thehive' in ansible_run_tags or 'start-keycloak' in ansible_run_tags or 'start-cortex' in ansible_run_tags or 'start-haproxy' in ansible_run_tags or 'start-cassandra' in ansible_run_tags or 'start-filebeat' in ansible_run_tags or 'start-misp' in ansible_run_tags or 'start-mysql' in ansible_run_tags or 'start-nifi' in ansible_run_tags or 'start-odfees' in ansible_run_tags or 'start-odfekibana' in ansible_run_tags" - name: Stop soctools cluster import_playbook: stopsoctools.yml diff --git a/startsoctools.yml b/startsoctools.yml index fec35b6b9396d64ef417ef1acd396a602d0d6622..2549876244bc4783a447f474c7a2a13a2a6c0313 100644 --- a/startsoctools.yml +++ b/startsoctools.yml @@ -1,62 +1,52 @@ --- -- name: Start docker containers - hosts: soctoolsmain - roles: - - docker - -- name: Reconfigure and start haproxy +- name: Start haproxy hosts: haproxy roles: - haproxy -- name: Reconfigure and start mysql +- name: Start mysql hosts: mysql roles: - mysql -- name: Reconfigure and start Cassandra +- name: Start Cassandra hosts: cassandra roles: - cassandra -- name: Reconfigure and start Keycloak +- name: Start Keycloak hosts: keycloakcontainers roles: - keycloak -- name: Reconfigure and start MISP +- name: Start MISP hosts: mispcontainers roles: - misp -- name: Reconfigure and start NiFi +- name: Start NiFi hosts: nificontainers roles: - nifi -- name: Reconfigure and start TheHive +- name: Start TheHive hosts: thehive roles: - thehive -- name: Reconfigure and start Cortex +- name: Start Cortex hosts: cortex roles: - cortex -- name: Reconfigure and start OpenDistro for Elasticsearch +- name: Start OpenDistro for Elasticsearch hosts: odfeescontainers roles: - odfees -- name: Reconfigure and start OpenDistro Kibana for Elasticsearch +- name: Start OpenDistro Kibana for Elasticsearch hosts: odfekibanacontainers roles: - odfekibana -- name: Install and run filebeat - hosts: filebeat - roles: - - filebeat - diff --git a/stopsoctools.yml b/stopsoctools.yml index 5bd121746729120486802cfc5538a181235b1642..8d28f6ed69ef08c23a008b8c36c653f811ec5783 100644 --- a/stopsoctools.yml +++ b/stopsoctools.yml @@ -5,8 +5,52 @@ roles: - nifi -- name: Stop all containers - hosts: soctoolsmain +- name: Stop haproxy + hosts: haproxy roles: - - docker + - haproxy +- name: Stop mysql + hosts: mysql + roles: + - mysql + +- name: Stop Cassandra + hosts: cassandra + roles: + - cassandra + +- name: Stop Keycloak + hosts: keycloakcontainers + roles: + - keycloak + +- name: Stop MISP + hosts: mispcontainers + roles: + - misp + +- name: Stop NiFi + hosts: nificontainers + roles: + - nifi + +- name: Stop TheHive + hosts: thehive + roles: + - thehive + +- name: Stop Cortex + hosts: cortex + roles: + - cortex + +- name: Stop OpenDistro for Elasticsearch + hosts: odfeescontainers + roles: + - odfees + +- name: Stop OpenDistro Kibana for Elasticsearch + hosts: odfekibanacontainers + roles: + - odfekibana