diff --git a/group_vars/all/main.yml b/group_vars/all/main.yml index feff4a6e61afa75ffac6ccb10543928eee8c8d23..c9fb1f4699229e9f03aa0b89381e48816cfe4abd 100644 --- a/group_vars/all/main.yml +++ b/group_vars/all/main.yml @@ -4,7 +4,7 @@ soctools_netname: "soctoolsnet" repo: gn43-dsl version: 7 -suffix: a20200528 +suffix: a20200612 temp_root: "/tmp/centosbuild" @@ -28,14 +28,13 @@ sysctlconfig: - { key: "net.core.wmem_max", val: "2097152" } - { key: "vm.max_map_count" , val: "524288" } -javamem: "384m" +nifi_javamem: "4g" +odfe_javamem: "2g" -ca_cn: "SOCTOOLS-CA" +nifi_version: 1.11.4 +nifi_repo: "https://archive.apache.org/dist" -#nifiadmin: -# - [ "Bozidar Proevski", "Pass001" ] -# - [ "Arne Oslebo", "Pass002" ] -# - [ "NifiELKuser", "Pass003" ] +ca_cn: "SOCTOOLS-CA" soctools_users: - firstname: "Bozidar" @@ -56,11 +55,12 @@ soctools_users: odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}" odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}" +# GENERATE 32-bit secure value odfekibana_cookie: "iroAm0ueIV7w6CS1WcJTwIV6R4d5RIAt" odfees_adminpass: "Pass004" #elk_version: "oss-7.6.1" -#odfeplugin_version: "1.7.0.0" elk_version: "oss-7.4.2" +#odfeplugin_version: "1.7.0.0" odfeplugin_version: "1.4.0.0" openid_realm: "SOCTOOLS1" diff --git a/roles/build/templates/nifi/Dockerfile.j2 b/roles/build/templates/nifi/Dockerfile.j2 index 2c0b69477fe0f5024070e7321e785d2fe561b5d4..d3408b16d5a52ad7cb48463cd1f543d2ab25c327 100644 --- a/roles/build/templates/nifi/Dockerfile.j2 +++ b/roles/build/templates/nifi/Dockerfile.j2 @@ -22,8 +22,8 @@ FROM {{repo}}/openjdk:{{version}}{{suffix}} ARG UID=1000 ARG GID=1000 -ARG NIFI_VERSION=1.11.4 -ARG BASE_URL=https://archive.apache.org/dist +ARG NIFI_VERSION={{nifi_version}} +ARG BASE_URL={{nifi_repo}} ARG MIRROR_BASE_URL=${MIRROR_BASE_URL:-${BASE_URL}} ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.zip} ARG NIFI_TOOLKIT_BINARY_PATH=${NIFI_TOOLKIT_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-toolkit-${NIFI_VERSION}-bin.zip} @@ -69,13 +69,13 @@ RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_BINARY_PATH} -o ${NIFI_BASE_DIR}/nifi-${ && mkdir -p ${NIFI_LOG_DIR} \ && ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} -VOLUME ${NIFI_LOG_DIR} \ - ${NIFI_HOME}/conf \ - ${NIFI_HOME}/database_repository \ - ${NIFI_HOME}/flowfile_repository \ - ${NIFI_HOME}/content_repository \ - ${NIFI_HOME}/provenance_repository \ - ${NIFI_HOME}/state +#VOLUME ${NIFI_LOG_DIR} \ +# ${NIFI_HOME}/conf \ +# ${NIFI_HOME}/database_repository \ +# ${NIFI_HOME}/flowfile_repository \ +# ${NIFI_HOME}/content_repository \ +# ${NIFI_HOME}/provenance_repository \ +# ${NIFI_HOME}/state # Clear nifi-env.sh in favour of configuring all environment variables in the Dockerfile RUN echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh diff --git a/roles/build/templates/odfees/Dockerfile-elastic.j2 b/roles/build/templates/odfees/Dockerfile-elastic.j2 index 3a51a784650298962e3cc9928d884a73a2fa8bf0..dd2ad126108e11ad523a8357393f1810de46d650 100644 --- a/roles/build/templates/odfees/Dockerfile-elastic.j2 +++ b/roles/build/templates/odfees/Dockerfile-elastic.j2 @@ -11,6 +11,8 @@ RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \ rpm -Uvh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{elk_version}}-no-jdk-x86_64.rpm && \ cp -a /etc/elasticsearch/ /usr/share/elasticsearch/config/ && \ chown -R elasticsearch /usr/share/elasticsearch/config && \ + mkdir -p /usr/share/elasticsearch/data && \ + chown -R elasticsearch /usr/share/elasticsearch/data && \ sed -i -e 's,ES_PATH_CONF=/etc/elasticsearch,ES_PATH_CONF=/usr/share/elasticsearch/config,g' /etc/sysconfig/elasticsearch EXPOSE 9200 9300 diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index a92bff6fa58493852c39f1031d5bd72316cbec83..31c2c0e74aca8a1ace93dae54ba51c78c80b6bd8 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,6 +1,7 @@ --- - include: networkcreate.yml +- include: volumecreate.yml - include: zookeeper.yml - include: nifi.yml - include: odfees.yml diff --git a/roles/docker/tasks/nifi.yml b/roles/docker/tasks/nifi.yml index 19ad16c03ccda44993c347f123cff718bf16efbd..7023444e9c0181dcde646dd0f1b4e55dc1f2b031 100644 --- a/roles/docker/tasks/nifi.yml +++ b/roles/docker/tasks/nifi.yml @@ -8,6 +8,8 @@ networks: - name: "{{ soctools_netname }}" networks_cli_compatible: yes + volumes: + - "{{item}}:/opt/nifi/nifi-current/conf" entrypoint: "/bin/bash" interactive: "yes" with_items: "{{ groups['nificontainers'] }}" diff --git a/roles/docker/tasks/odfees.yml b/roles/docker/tasks/odfees.yml index a159eadc3382254b6877bfdd32acc5c4c238d29e..d54ebd59216d12b7c42e15005b3f6b15fb9affa9 100644 --- a/roles/docker/tasks/odfees.yml +++ b/roles/docker/tasks/odfees.yml @@ -8,6 +8,8 @@ networks: - name: "{{ soctools_netname }}" networks_cli_compatible: yes + volumes: + - "{{item}}:/usr/share/elasticsearch/data" entrypoint: "/bin/bash" interactive: "yes" with_items: "{{ groups['odfeescontainers'] }}" diff --git a/roles/docker/tasks/volumecreate.yml b/roles/docker/tasks/volumecreate.yml new file mode 100644 index 0000000000000000000000000000000000000000..cb131af09995fb26b722a43b369bf877fca7455f --- /dev/null +++ b/roles/docker/tasks/volumecreate.yml @@ -0,0 +1,18 @@ +--- + +- name: Create NiFi volumes + docker_volume: + name: "{{item}}" + with_items: + - "{{ groups['nificontainers'] }}" + tags: + - start + +- name: Create OpenDistro for Elasticearch volumes + docker_volume: + name: "{{item}}" + with_items: + - "{{ groups['odfeescontainers'] }}" + tags: + - start + diff --git a/roles/keycloak/tasks/main.yml b/roles/keycloak/tasks/main.yml index 1de65c9d200e08c60c2f26c6967a1509bd42c82b..9c8f81e6635ee43d57a90866accd961304e14760 100644 --- a/roles/keycloak/tasks/main.yml +++ b/roles/keycloak/tasks/main.yml @@ -1,13 +1,5 @@ --- -#- name: Create config directory -# file: -# name: config -# state: directory -# mode: 0700 -# tags: -# - start - - name: Copy certificates in keycloak x509 conf dir copy: src: "{{ item.local }}" @@ -52,23 +44,12 @@ tags: - start -#- name: Exit here to test ODFE -# meta: end_play -# tags: -# - start - name: Start Keycloak IdP command: /opt/jboss/tools/startkeycloak.sh - #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml & - #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &" tags: - start -#- name: Exit here to test Keycloak -# meta: end_play -# tags: -# - start - - name: Wait for Keycloak wait_for: host: "{{groups['keycloakcontainers'][0]}}" @@ -78,13 +59,6 @@ tags: - start -#- name: Start Keycloak -# command: /opt/jboss/keycloak/bin/standalone.sh -b 0.0.0.0 -# #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml & -# #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &" -# tags: -# - start - - name: Initialize Keycloak realm command: /opt/jboss/tools/initkeycloakrealm.sh tags: @@ -103,11 +77,6 @@ tags: - start -#- name: Exit here to test Keycloak -# meta: end_play -# tags: -# - start - - name: Stop Keycloak command: "pkill -SIGTERM -F {{inventory_hostname}}.pid" tags: diff --git a/roles/nifi/templates/bootstrap.conf.j2 b/roles/nifi/templates/bootstrap.conf.j2 index 4b92c44d8610e7291762a55674ada2be6d18b549..b991f4b24101fdd9472f4c33fd3953736bd862ab 100644 --- a/roles/nifi/templates/bootstrap.conf.j2 +++ b/roles/nifi/templates/bootstrap.conf.j2 @@ -32,8 +32,8 @@ graceful.shutdown.seconds=20 java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true # JVM memory settings -java.arg.2=-Xms{{ javamem }} -java.arg.3=-Xmx{{ javamem }} +java.arg.2=-Xms{{ nifi_javamem }} +java.arg.3=-Xmx{{ nifi_javamem }} # Enable Remote Debugging #java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 diff --git a/roles/odfees/tasks/main.yml b/roles/odfees/tasks/main.yml index 87ee2a129eb0d15797436ab99243448e094f4c2e..d0f40ce3de42100ba1fc9185c0e36ea02ab608b0 100644 --- a/roles/odfees/tasks/main.yml +++ b/roles/odfees/tasks/main.yml @@ -14,14 +14,6 @@ tags: - start -- name: Create config directory - file: - name: config - state: directory - mode: 0700 - tags: - - start - - name: Copy certificates in odfe conf dir copy: src: "{{ item }}" @@ -46,20 +38,6 @@ tags: - start -# lineinfile: -# path: /etc/sysconfig/elasticsearch -# regexp: '^ES_PATH_CONF=' -# line: ES_PATH_CONF=/usr/share/elasticsearch/config -# tags: -# - start - - -#- name: Get openid authkey -# set_fact: -# odfesecret: "{{lookup('file', 'files/odfesecret',convert_data=False) | from_json }}" -# tags: -# - start - - name: Configure odfe properties template: src: "config/{{item}}.j2" @@ -88,7 +66,6 @@ template: src: "securityconfig/{{item}}.j2" dest: "plugins/opendistro_security/securityconfig/{{item}}" - #dest: "{{item}}" with_items: - internal_users.yml - config.yml @@ -121,16 +98,6 @@ tags: - start -#- name: check reachable hosts -# gather_facts: no -# tasks: -# - command: ping -c1 {{ inventory_hostname }} -# delegate_to: localhost -# register: ping_result -# ignore_errors: yes -# - group_by: key=reachable -# when: ping_result|success - - name: Stop OpenDistro for Elasticsearch command: "pkill -SIGTERM -F {{inventory_hostname}}.pid" tags: diff --git a/roles/odfees/templates/config/jvm.options.j2 b/roles/odfees/templates/config/jvm.options.j2 index 8717fc8f40604981baeffa02970ae213145f78e5..d2a22392210eadbdf65de97c77fd427799753c6a 100644 --- a/roles/odfees/templates/config/jvm.options.j2 +++ b/roles/odfees/templates/config/jvm.options.j2 @@ -19,8 +19,8 @@ # Xms represents the initial size of total heap space # Xmx represents the maximum size of total heap space --Xms512m --Xmx512m +-Xms{{odfe_javamem}} +-Xmx{{odfe_javamem}} ################################################################ ## Expert settings