diff --git a/HOWTOS.md b/HOWTOS.md deleted file mode 100644 index ae4e2da7b96e93025bcb526dc317b2e7ff918896..0000000000000000000000000000000000000000 --- a/HOWTOS.md +++ /dev/null @@ -1,73 +0,0 @@ -Howto's -======= - -Modify main NiFi pipeline -------------------------- - -To make modifications to the main NiFi pipeline and add it to the Ansible playbook, do the following in the soctool directory: - -* Make necesarry to the pipeline in the NiFi GUI -* Copy flow.xml.gz file from one of the NiFi containers: - `docker cp soctools-nifi-1:/opt/nifi/nifi-current/conf/flow.xml.gz .` -* Convert flowx.xml.gz to new template - `utils/flow2template.py flow.xml.gz roles/nifi/templates/flow.xml.j2` - - -Update configuration files in docker containers using Ansible -------------------------------------------------------------- -To update configuration files for all docker containers together, run the following command: - ansible-playbook -i inventories soctools.yml -t update-config -To update configuration files only for specific services, run the following commands: - ansible-playbook -i inventories soctools.yml -t update-keycloak-config - ansible-playbook -i inventories soctools.yml -t update-thehive-config - ansible-playbook -i inventories soctools.yml -t update-cortex-config - ansible-playbook -i inventories soctools.yml -t update-cassandra-config - ansible-playbook -i inventories soctools.yml -t update-haproxy-config - ansible-playbook -i inventories soctools.yml -t update-filebeat-config - ansible-playbook -i inventories soctools.yml -t update-nifi-config - ansible-playbook -i inventories soctools.yml -t update-odfees-config - ansible-playbook -i inventories soctools.yml -t update-odfekibana-config - - -Restart services inside docker containers using Ansible -------------------------------------------------------- -To restart services for all docker containers together, run the following command: - ansible-playbook -i inventories soctools.yml -t restart -To restart services only for specific docker containers, run the following commands: - ansible-playbook -i inventories soctools.yml -t restart-keycloak - ansible-playbook -i inventories soctools.yml -t restart-thehive - ansible-playbook -i inventories soctools.yml -t restart-cortex - ansible-playbook -i inventories soctools.yml -t restart-cassandra - ansible-playbook -i inventories soctools.yml -t restart-haproxy - ansible-playbook -i inventories soctools.yml -t restart-filebeat - ansible-playbook -i inventories soctools.yml -t restart-misp - ansible-playbook -i inventories soctools.yml -t restart-mysql - ansible-playbook -i inventories soctools.yml -t restart-nifi - ansible-playbook -i inventories soctools.yml -t restart-odfees - ansible-playbook -i inventories soctools.yml -t restart-odfekibana - -Stop services inside docker containers using Ansible ----------------------------------------------------- -To stop services for all docker containers together, run the following command: - ansible-playbook -i inventories soctools.yml -t stop -To stop services only for specific docker containers, run the following commands: - ansible-playbook -i inventories soctools.yml -t stop-keycloak - ansible-playbook -i inventories soctools.yml -t stop-thehive - ansible-playbook -i inventories soctools.yml -t stop-cortex - ansible-playbook -i inventories soctools.yml -t stop-cassandra - ansible-playbook -i inventories soctools.yml -t stop-haproxy - ansible-playbook -i inventories soctools.yml -t stop-filebeat - ansible-playbook -i inventories soctools.yml -t stop-misp - ansible-playbook -i inventories soctools.yml -t stop-mysql - ansible-playbook -i inventories soctools.yml -t stop-nifi - ansible-playbook -i inventories soctools.yml -t stop-odfees - ansible-playbook -i inventories soctools.yml -t stop-odfekibana - -Restart services inside docker containers manually --------------------------------------------------- -To restart services inside docker containers after changes in configuration files: - 1. Attache container: docker exec -it container_id_or_name bash (example: docker exec -it soctools-keycloak bash) - 2. List services and their statuses: supervisorctl status - 3. Restart service: supervisorctl restart supervisor_service_name (example: supervisorctl restart keycloak) - 4. Detach from container: exit - diff --git a/razliki b/razliki deleted file mode 100644 index 10e6a9e7ce17525e689c4ff4546b9f6cd8a2962c..0000000000000000000000000000000000000000 --- a/razliki +++ /dev/null @@ -1,466 +0,0 @@ -diff --git a/group_vars/all/main.yml b/group_vars/all/main.yml -index 6bb820d..c6adf5f 100644 ---- a/group_vars/all/main.yml -+++ b/group_vars/all/main.yml -@@ -4,8 +4,32 @@ dslproxy: "dsoclab.gn4-3-wp8-soc.sunet.se" - - # TheHive Button plugin - THEHIVE_URL: "https://hive.gn4-3-wp8-soc.sunet.se/" --THEHIVE_API_KEY: "5LymseWiurZBrQN8Kqp8O+9KniTL5cE0" --THEHIVE_OWNER: "admin" -+# here enter API key for default admin user -+THEHIVE_API_KEY: "bs2Jc3tGJqhVv0AYyX2NYlhMlorPz7mX" -+# ID of the default admin user -+THEHIVE_OWNER: "admin@thehive.local" -+ -+# TheHive Create Organisation and Users -+# Login as default admin user and create API key, populate it here -+# thehive_admin_api: "KoHrKbIJm8XMsJxA9nZLs6YemCu76o3u" -+# thehive_writer: "[write]" -+ -+#THEHIVE_API_KEY: "1gFdNhmUSxO3BRe1SBB5JYEvkW9UOo6s" -+THEHIVE_USERS: -+ - kiril: -+ username: "kiril" -+ name: "Kiril" -+ surname: "Kiroski" -+ roles: '["read", "write", "admin"]' -+ organization: "uninett.no" -+ - temur: -+ username: "temur" -+ name: "Temur" -+ surname: "Maisuradze" -+ roles: '["read", "write", "admin"]' -+ organization: "uninett.no" -+ -+ - - soctools_netname: "soctoolsnet" - soctools_network: "172.22.0.0/16" -@@ -82,6 +106,13 @@ soctools_users: - DN: "CN=Arne Oslebo" - CN: "Arne Oslebo" - password: "Pass002" -+ - firstname: "Kiril" -+ lastname: "Kjiroski" -+ username: "kiril.kjiroski" -+ email: "kiril.kjiroski@finki.ukim.mk" -+ DN: "CN=Kiril Kjiroski" -+ CN: "Kiril Kjiroski" -+ password: "Pass003" - - odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}" - odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}" -diff --git a/roles/ca/tasks/main.yml b/roles/ca/tasks/main.yml -index ec25dad..6ca350a 100644 ---- a/roles/ca/tasks/main.yml -+++ b/roles/ca/tasks/main.yml -@@ -229,6 +229,7 @@ - - keycloak - - misp - - cortex -+ - thehive - - - name: Copy ca cert to roles - copy: -diff --git a/roles/cortex/tasks/main.yml b/roles/cortex/tasks/main.yml -index 5d1eeb2..06b2639 100644 ---- a/roles/cortex/tasks/main.yml -+++ b/roles/cortex/tasks/main.yml -@@ -31,6 +31,12 @@ - - start - - startcortex - -+- name: Get openid authkey -+ set_fact: -+ cortexsecret: "{{lookup('file', 'files/cortexsecret',convert_data=False) | from_json }}" -+ tags: -+ - start -+ - - name: Configure embedded Elasticsearch 6 - remote_user: root - template: -@@ -61,6 +67,13 @@ - - start - - startcortex - -+- name: Configure Cortex logging -+ copy: -+ src: logback.xml -+ dest: /etc/cortex/logback.xml -+ tags: -+ - start -+ - - name: Start Cortex - command: > - daemonize -diff --git a/roles/cortex/templates/application.conf.j2 b/roles/cortex/templates/application.conf.j2 -index 35323e0..6d6d09c 100644 ---- a/roles/cortex/templates/application.conf.j2 -+++ b/roles/cortex/templates/application.conf.j2 -@@ -66,7 +66,7 @@ auth { - # the "ad" section below. - # - ldap : use LDAP to authenticate users. The associated configuration shall be done in the - # "ldap" section below. -- provider = [local] -+ provider = [local,oauth2] - - ad { - # The Windows domain name in DNS format. This parameter is required if you do not use -@@ -108,6 +108,84 @@ auth { - # If 'true', use SSL to connect to the LDAP directory server. - #useSSL = true - } -+ oauth2 { -+ # URL of the authorization server -+ clientId = "dsoclab-cortex" -+ clientSecret = {{cortexsecret.value}} -+ redirectUri = "https://{{dslproxy}}:9001/api/ssoLogin" -+ responseType = "code" -+ grantType = "authorization_code" -+ -+ # URL from where to get the access token -+ authorizationUrl = "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/auth" -+ authorizationHeader = "Bearer" -+ tokenUrl = "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/token" -+ -+ -+ # The endpoint from which to obtain user details using the OAuth token, after successful login -+ userUrl = "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/userinfo" -+ scope = "profile" -+ userIdField = "email" -+ #userUrl = "https://auth-site.com/api/User" -+ #scope = ["openid profile"] -+ } -+ -+ ws.ssl.trustManager { -+ stores = [ -+ { -+ type = "JKS" // JKS or PEM -+ path = "cacerts.jks" -+ password = "{{tspass}}" -+ } -+ ] -+ } -+ -+ -+ # Single-Sign On -+ sso { -+ # Autocreate user in database? -+ autocreate = true -+ -+ # Autoupdate its profile and roles? -+ autoupdate = true -+ -+ # Autologin user using SSO? -+ autologin = true -+ -+ # Name of mapping class from user resource to backend user ('simple' or 'group') -+ #mapper = group -+ #mapper = simple -+ #attributes { -+ # login = "user" -+ # name = "name" -+ # groups = "groups" -+ # organization = "org" -+ #} -+# defaultRoles = ["read", "write", "admin"] -+# defaultOrganization = "uninett.no" -+ #defaultRoles = ["read"] -+ #defaultOrganization = "csirt" -+ #groups { -+ # # URL to retreive groups (leave empty if you are using OIDC) -+ # #url = "https://auth-site.com/api/Groups" -+ # # Group mappings, you can have multiple roles for each group: they are merged -+ # mappings { -+ # admin-profile-name = ["admin"] -+ # editor-profile-name = ["write"] -+ # reader-profile-name = ["read"] -+ # } -+ #} -+ -+ mapper = simple -+ attributes { -+ login = "user" -+ name = "name" -+ roles = "roles" -+ organization = "org" -+ } -+ defaultRoles = ["read", "analyze"] -+ defaultOrganization = "uninett.no" -+ } - } - - ## ANALYZERS -diff --git a/roles/docker/tasks/thehive.yml b/roles/docker/tasks/thehive.yml -index f8effea..30b11c8 100644 ---- a/roles/docker/tasks/thehive.yml -+++ b/roles/docker/tasks/thehive.yml -@@ -15,6 +15,7 @@ - with_items: "{{ groups['thehive'] }}" - tags: - - start -+ - thehivestart - - - name: Disconnect thehive containers from network and remove - docker_container: -@@ -23,4 +24,4 @@ - with_items: "{{ groups['thehive'] }}" - tags: - - stop -- -+ - thehivestop -diff --git a/roles/keycloak/tasks/main.yml b/roles/keycloak/tasks/main.yml -index 9c8f81e..2bb6a62 100644 ---- a/roles/keycloak/tasks/main.yml -+++ b/roles/keycloak/tasks/main.yml -@@ -4,7 +4,7 @@ - copy: - src: "{{ item.local }}" - dest: "{{ item.remote }}" -- mode: "{{ item.mode}}" -+ mode: "{{ item.mode }}" - with_items: - - local: "files/{{ inventory_hostname }}.crt" - remote: /etc/x509/https/tls.crt -@@ -20,6 +20,7 @@ - mode: '0644' - tags: - - start -+ - startkeycloak - - - name: Generate Keycloak secure config - command: "/opt/jboss/tools/x509.sh" -@@ -27,11 +28,14 @@ - X509_CA_BUNDLE: "/etc/x509/ca/ca.crt" - tags: - - start -+ - startkeycloak - - - name: Set admin password - command: /opt/jboss/keycloak/bin/add-user-keycloak.sh --user "admin" --password "{{keycloak_adminpass}}" -+ ignore_errors: yes - tags: - - start -+ - startkeycloak - - - name: Configure Keycloak start script - template: -@@ -43,12 +47,14 @@ - - initkeycloakrealm.sh - tags: - - start -+ - startkeycloak - - - - name: Start Keycloak IdP - command: /opt/jboss/tools/startkeycloak.sh - tags: - - start -+ - startkeycloak - - - name: Wait for Keycloak - wait_for: -@@ -58,11 +64,13 @@ - delay: 5 - tags: - - start -+ - startkeycloak - - - name: Initialize Keycloak realm - command: /opt/jboss/tools/initkeycloakrealm.sh - tags: - - start -+ - startkeycloak - - - name: Copy secrets from Keycloak - fetch: -@@ -74,10 +82,16 @@ - local: "roles/nifi/files/nifisecret" - - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/kibanasecret" - local: "roles/odfekibana/files/kibanasecret" -+ - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/thehivesecret" -+ local: "roles/thehive/files/thehivesecret" -+ - remote: "{{ ansible_facts.env['JBOSS_HOME'] }}/cortexsecret" -+ local: "roles/cortex/files/cortexsecret" - tags: - - start -+ - startkeycloak - - - name: Stop Keycloak - command: "pkill -SIGTERM -F {{inventory_hostname}}.pid" - tags: - - stop -+ - stopkeycloak -diff --git a/roles/keycloak/templates/initkeycloakrealm.sh.j2 b/roles/keycloak/templates/initkeycloakrealm.sh.j2 -index f3f0073..d6fc946 100644 ---- a/roles/keycloak/templates/initkeycloakrealm.sh.j2 -+++ b/roles/keycloak/templates/initkeycloakrealm.sh.j2 -@@ -28,6 +28,12 @@ kcadm.sh get realms/{{openid_realm}}/clients/${NIFICLIENT}/client-secret --field - KIBANACLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-kibana","protocol":"openid-connect","clientAuthenticatorType": "client-secret","rootUrl": "https://{{dslproxy}}:5601","adminUrl": "","redirectUris": ["https://{{dslproxy}}:5601", "https://{{dslproxy}}:5601/auth/openid/login", "https://{{dslproxy}}:5601/app/kibana" ],"webOrigins": [], "publicClient": false }') - kcadm.sh get realms/{{openid_realm}}/clients/${KIBANACLIENT}/client-secret --fields value > /opt/jboss/keycloak/kibanasecret - -+THEHIVECLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-thehive","protocol":"openid-connect","clientAuthenticatorType": "client-secret","adminUrl": "","redirectUris": ["https://{{dslproxy}}:9000/api/ssoLogin"],"webOrigins": [], "publicClient": false }') -+kcadm.sh get realms/{{openid_realm}}/clients/${THEHIVECLIENT}/client-secret --fields value > /opt/jboss/keycloak/thehivesecret -+ -+CORTEXCLIENT=$(kcadm.sh create realms/{{openid_realm}}/clients -i -b '{"enabled":true, "clientId":"dsoclab-cortex","protocol":"openid-connect","clientAuthenticatorType": "client-secret","adminUrl": "","redirectUris": ["https://{{dslproxy}}:9001/api/ssoLogin"],"webOrigins": [], "publicClient": false }') -+kcadm.sh get realms/{{openid_realm}}/clients/${CORTEXCLIENT}/client-secret --fields value > /opt/jboss/keycloak/cortexsecret -+ - - kcadm.sh config truststore --delete - -diff --git a/roles/thehive/tasks/main.yml b/roles/thehive/tasks/main.yml -index 7d8f859..0e560e7 100644 ---- a/roles/thehive/tasks/main.yml -+++ b/roles/thehive/tasks/main.yml -@@ -1,5 +1,39 @@ - --- - -+- name: Copy cacert to ca-trust dir -+ remote_user: root -+ copy: -+ src: "files/{{ca_cn}}.crt" -+ dest: /etc/pki/ca-trust/source/anchors/ca.crt -+ tags: -+ - start -+ -+- name: Install cacert to root truststore -+ remote_user: root -+ command: "update-ca-trust" -+ tags: -+ - start -+ -+- name: Copy certificates in thehive conf dir -+ copy: -+ src: "{{ item }}" -+ dest: "/etc/thehive/{{ item }}" -+ mode: 0600 -+ with_items: -+ - "{{ inventory_hostname }}.crt" -+ - "{{ inventory_hostname }}.key" -+ - cacerts.jks -+ - "{{ca_cn}}.crt" -+ tags: -+ - start -+ -+- name: Get openid authkey -+ set_fact: -+ thehivesecret: "{{lookup('file', 'files/thehivesecret',convert_data=False) | from_json }}" -+ tags: -+ - start -+ -+ - - name: Configure TheHive - template: - src: application.conf.j2 -@@ -7,6 +41,14 @@ - tags: - - start - -+- name: Configure TheHive logging -+ copy: -+ src: logback.xml -+ dest: /etc/thehive/logback.xml -+ tags: -+ - start -+ -+ - - name: Start TheHive - command: > - daemonize -@@ -31,8 +73,15 @@ - tags: - - start - -+- name: Create TheHive users -+ include: createusers.yml -+ tags: -+ - createusers -+ - start -+ - - name: Stop TheHive - command: "pkill -SIGTERM -F /tmp/thehive.pid" - tags: - - stop -+ - stopthehive - -diff --git a/roles/thehive/templates/application.conf.j2 b/roles/thehive/templates/application.conf.j2 -index 6fa36eb..a92e4f7 100644 ---- a/roles/thehive/templates/application.conf.j2 -+++ b/roles/thehive/templates/application.conf.j2 -@@ -13,7 +13,7 @@ db.janusgraph { - ## Cassandra configuration - # More information at https://docs.janusgraph.org/basics/configuration-reference/#storagecql - backend: cql -- hostname: ["{{groups['cassandra'][0]}}.{{soctools_netname}}"] -+ hostname: ["{{groups['cassandra'][0]}}.{{soctools_netname}}:9042"] - # Cassandra authentication (if configured) - // username: "thehive" - // password: "password" -@@ -47,17 +47,61 @@ storage { - - ## Authentication configuration - # More information at https://github.com/TheHive-Project/TheHiveDocs/TheHive4/Administration/Authentication.md --//auth { --// providers: [ -+auth { -+ providers: [ - // {name: session} # required ! - // {name: basic, realm: thehive} - // {name: local} - // {name: key} --// ] -+ {name: session} # required ! -+ {name: basic, realm: thehive} -+ {name: local} -+ {name: key} -+ { -+ name: oauth2 -+ clientId: "dsoclab-thehive" -+ clientSecret: {{thehivesecret.value}} -+ redirectUri: "https://{{dslproxy}}:9000/api/ssoLogin" -+ responseType: "code" -+ grantType: "authorization_code" -+ authorizationUrl: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/auth" -+ authorizationHeader: "Bearer" -+ tokenUrl: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/token" -+ userUrl: "https://{{dslproxy}}:12443/auth/realms/{{openid_realm}}/protocol/openid-connect/userinfo" -+// scope: ["openid", "email"] -+ scope: ["openid"] -+ userIdField: "email" -+// userIdField: "name" -+ } -+ ] -+ sso { -+ autocreate: true -+ autoupdate: true -+ autologin: true -+ mapper: "simple" -+// attributes { -+// login: "login" -+// name: "name" -+// roles: "role" -+// } -+ defaultRoles: ["read", "write", "admin"] -+ defaultOrganization: "uninett.no" -+// defaultOrganization: "demo" -+ } -+ ws.ssl.trustManager { -+ stores = [ -+ { -+ type: "JKS" // JKS or PEM -+ path: "cacerts.jks" -+ password: "{{tspass}}" -+ } -+ ] -+ } - # The format of logins must be valid email address format. If the provided login doesn't contain `@` the following - # domain is automatically appended --// defaultUserDomain: "thehive.local" --//} -+ defaultUserDomain: "uninett.no" -+# defaultUserDomain: "thehive.local" -+} - - ## CORTEX configuration - # More information at https://github.com/TheHive-Project/TheHiveDocs/TheHive4/Administration/Connectors.md diff --git a/restart-soctools.yml b/restart-soctools.yml deleted file mode 100644 index 5aa1c94116243ecdf120c17813677b9c59643630..0000000000000000000000000000000000000000 --- a/restart-soctools.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- - -- name: Restart services for haproxy - hosts: haproxy - roles: - - haproxy - -- name: Restart services for mysql - hosts: mysql - roles: - - mysql - -- name: Restart services for Cassandra - hosts: cassandra - roles: - - cassandra - -- name: Restart services for Keycloak - hosts: keycloakcontainers - roles: - - keycloak - -- name: Restart services for NiFi - hosts: nificontainers - roles: - - nifi - -- name: Restart services for OpenDistro for Elasticsearch - hosts: odfeescontainers - roles: - - odfees - -- name: Restart services for OpenDistro Kibana for Elasticsearch - hosts: odfekibanacontainers - roles: - - odfekibana - -- name: Restart services for MISP - hosts: mispcontainers - roles: - - misp - -- name: Restart services for TheHive - hosts: thehive - roles: - - thehive - -- name: Restart services for Cortex - hosts: cortex - roles: - - cortex - diff --git a/update-config-soctools.yml b/update-config-soctools.yml deleted file mode 100644 index 138f664fee2eb1a9d882a7ebf0b8f9e390bb1e72..0000000000000000000000000000000000000000 --- a/update-config-soctools.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- - -- name: Update Configs for haproxy - hosts: haproxy - roles: - - haproxy - -- name: Update Configs for mysql - hosts: mysql - roles: - - mysql - -- name: Update Configs for Cassandra - hosts: cassandra - roles: - - cassandra - -- name: Update Configs for Keycloak - hosts: keycloakcontainers - roles: - - keycloak - -- name: Update Configs for NiFi - hosts: nificontainers - roles: - - nifi - -- name: Update Configs for OpenDistro for Elasticsearch - hosts: odfeescontainers - roles: - - odfees - -- name: Update Configs for OpenDistro Kibana for Elasticsearch - hosts: odfekibanacontainers - roles: - - odfekibana - -- name: Update Configs for MISP - hosts: mispcontainers - roles: - - misp - -- name: Update Configs for TheHive - hosts: thehive - roles: - - thehive - -- name: Update Configs for Cortex - hosts: cortex - roles: - - cortex -