Memlimits and volumes

Added memlimits for NiFi and ODFE
Added Docker volumes configuration for NiFi and ODFE
Some code cleanup

group_vars/all/main.yml

@@ -4,7 +4,7 @@ soctools_netname: "soctoolsnet"
 
 repo: gn43-dsl
 version: 7
-suffix: a20200528
+suffix: a20200612
 
 temp_root: "/tmp/centosbuild"
 
@@ -28,14 +28,13 @@ sysctlconfig:
   - { key: "net.core.wmem_max", val: "2097152" }
   - { key: "vm.max_map_count" , val:  "524288" }
 
-javamem: "384m"
+nifi_javamem: "4g"
+odfe_javamem: "2g"
 
-ca_cn: "SOCTOOLS-CA"
+nifi_version: 1.11.4
+nifi_repo: "https://archive.apache.org/dist"
 
-#nifiadmin:
-#  - [ "Bozidar Proevski", "Pass001" ]
-#  - [ "Arne Oslebo", "Pass002" ]
-#  - [ "NifiELKuser", "Pass003" ]
+ca_cn: "SOCTOOLS-CA"
 
 soctools_users:
   - firstname: "Bozidar"
@@ -56,11 +55,12 @@ soctools_users:
 
 odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}"
 odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}"
+# GENERATE 32-bit secure value
 odfekibana_cookie: "iroAm0ueIV7w6CS1WcJTwIV6R4d5RIAt"
 odfees_adminpass: "Pass004"
 #elk_version: "oss-7.6.1"
-#odfeplugin_version: "1.7.0.0"
 elk_version: "oss-7.4.2"
+#odfeplugin_version: "1.7.0.0"
 odfeplugin_version: "1.4.0.0"
 
 openid_realm: "SOCTOOLS1"

roles/build/templates/nifi/Dockerfile.j2

@@ -22,8 +22,8 @@ FROM {{repo}}/openjdk:{{version}}{{suffix}}
 
 ARG UID=1000
 ARG GID=1000
-ARG NIFI_VERSION=1.11.4
-ARG BASE_URL=https://archive.apache.org/dist
+ARG NIFI_VERSION={{nifi_version}}
+ARG BASE_URL={{nifi_repo}}
 ARG MIRROR_BASE_URL=${MIRROR_BASE_URL:-${BASE_URL}}
 ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.zip}
 ARG NIFI_TOOLKIT_BINARY_PATH=${NIFI_TOOLKIT_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-toolkit-${NIFI_VERSION}-bin.zip}
@@ -69,13 +69,13 @@ RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_BINARY_PATH} -o ${NIFI_BASE_DIR}/nifi-${
     && mkdir -p ${NIFI_LOG_DIR} \
     && ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}
 
-VOLUME ${NIFI_LOG_DIR} \
-       ${NIFI_HOME}/conf \
-       ${NIFI_HOME}/database_repository \
-       ${NIFI_HOME}/flowfile_repository \
-       ${NIFI_HOME}/content_repository \
-       ${NIFI_HOME}/provenance_repository \
-       ${NIFI_HOME}/state
+#VOLUME ${NIFI_LOG_DIR} \
+#       ${NIFI_HOME}/conf \
+#       ${NIFI_HOME}/database_repository \
+#       ${NIFI_HOME}/flowfile_repository \
+#       ${NIFI_HOME}/content_repository \
+#       ${NIFI_HOME}/provenance_repository \
+#       ${NIFI_HOME}/state
 
 # Clear nifi-env.sh in favour of configuring all environment variables in the Dockerfile
 RUN echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh

roles/build/templates/odfees/Dockerfile-elastic.j2

@@ -11,6 +11,8 @@ RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
     rpm -Uvh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{elk_version}}-no-jdk-x86_64.rpm && \
     cp -a /etc/elasticsearch/ /usr/share/elasticsearch/config/ && \
     chown -R elasticsearch /usr/share/elasticsearch/config && \
+    mkdir -p /usr/share/elasticsearch/data && \
+    chown -R elasticsearch /usr/share/elasticsearch/data && \
     sed -i -e 's,ES_PATH_CONF=/etc/elasticsearch,ES_PATH_CONF=/usr/share/elasticsearch/config,g' /etc/sysconfig/elasticsearch
 
 EXPOSE 9200 9300

roles/docker/tasks/main.yml

 ---
 
 - include: networkcreate.yml
+- include: volumecreate.yml
 - include: zookeeper.yml
 - include: nifi.yml
 - include: odfees.yml

roles/docker/tasks/nifi.yml

@@ -8,6 +8,8 @@
     networks:
       - name: "{{ soctools_netname }}"
     networks_cli_compatible: yes
+    volumes:
+      - "{{item}}:/opt/nifi/nifi-current/conf"
     entrypoint: "/bin/bash"
     interactive: "yes"
   with_items: "{{ groups['nificontainers'] }}"

roles/docker/tasks/odfees.yml

@@ -8,6 +8,8 @@
     networks:
       - name: "{{ soctools_netname }}"
     networks_cli_compatible: yes
+    volumes:
+      - "{{item}}:/usr/share/elasticsearch/data"
     entrypoint: "/bin/bash"
     interactive: "yes"
   with_items: "{{ groups['odfeescontainers'] }}"

roles/docker/tasks/volumecreate.yml

+---
+
+- name: Create NiFi volumes
+  docker_volume:
+    name: "{{item}}"
+  with_items:
+    - "{{ groups['nificontainers'] }}" 
+  tags:
+    - start
+
+- name: Create OpenDistro for Elasticearch volumes
+  docker_volume:
+    name: "{{item}}"
+  with_items:
+    - "{{ groups['odfeescontainers'] }}" 
+  tags:
+    - start
+

roles/keycloak/tasks/main.yml

 ---
 
-#- name: Create config directory
-#  file:
-#    name: config
-#    state: directory
-#    mode: 0700
-#  tags:
-#    - start
-
 - name: Copy certificates in keycloak x509 conf dir
   copy:
     src:  "{{ item.local }}"
@@ -52,23 +44,12 @@
   tags:
     - start
 
-#- name: Exit here to test ODFE
-#  meta: end_play
-#  tags:
-#    - start
 
 - name: Start Keycloak IdP
   command: /opt/jboss/tools/startkeycloak.sh
-  #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml &
-  #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &"
   tags:
     - start
 
-#- name: Exit here to test Keycloak
-#  meta: end_play
-#  tags:
-#    - start
-
 - name: Wait for Keycloak
   wait_for:
     host: "{{groups['keycloakcontainers'][0]}}"
@@ -78,13 +59,6 @@
   tags:
     - start
 
-#- name: Start Keycloak
-#  command: /opt/jboss/keycloak/bin/standalone.sh -b 0.0.0.0 
-#  #shell: exec /usr/share/kibana/bin/kibana -c config/kibana.yml &
-#  #shell: "nohup /usr/share/kibana/bin/kibana -c config/kibana.yml &"
-#  tags:
-#    - start
-
 - name: Initialize Keycloak realm
   command: /opt/jboss/tools/initkeycloakrealm.sh
   tags:
@@ -103,11 +77,6 @@
   tags:
     - start
 
-#- name: Exit here to test Keycloak
-#  meta: end_play
-#  tags:
-#    - start
-
 - name: Stop Keycloak
   command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
   tags:

roles/nifi/templates/bootstrap.conf.j2

@@ -32,8 +32,8 @@ graceful.shutdown.seconds=20
 java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
 
 # JVM memory settings
-java.arg.2=-Xms{{ javamem }}
-java.arg.3=-Xmx{{ javamem }}
+java.arg.2=-Xms{{ nifi_javamem }}
+java.arg.3=-Xmx{{ nifi_javamem }}
 
 # Enable Remote Debugging
 #java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000

roles/odfees/tasks/main.yml

@@ -14,14 +14,6 @@
   tags:
     - start
 
-- name: Create config directory
-  file:
-    name: config
-    state: directory
-    mode: 0700
-  tags:
-    - start
-
 - name: Copy certificates in odfe conf dir
   copy:
     src:  "{{ item }}"
@@ -46,20 +38,6 @@
   tags: 
     - start
 
-#  lineinfile:
-#    path: /etc/sysconfig/elasticsearch
-#    regexp: '^ES_PATH_CONF='
-#    line: ES_PATH_CONF=/usr/share/elasticsearch/config
-#  tags:
-#    - start
-
-
-#- name: Get openid authkey
-#  set_fact:
-#    odfesecret: "{{lookup('file', 'files/odfesecret',convert_data=False) | from_json }}"
-#  tags:
-#    - start
-
 - name: Configure odfe properties
   template:
     src: "config/{{item}}.j2"
@@ -88,7 +66,6 @@
   template:
     src: "securityconfig/{{item}}.j2"
     dest: "plugins/opendistro_security/securityconfig/{{item}}"
-    #dest: "{{item}}"
   with_items:
     - internal_users.yml
     - config.yml
@@ -121,16 +98,6 @@
   tags:
     - start
 
-#- name: check reachable hosts
-#  gather_facts: no
-#  tasks:
-#    - command: ping -c1 {{ inventory_hostname }}
-#      delegate_to: localhost
-#      register: ping_result
-#      ignore_errors: yes
-#    - group_by: key=reachable
-#      when: ping_result|success
-
 - name: Stop OpenDistro for Elasticsearch
   command: "pkill -SIGTERM -F {{inventory_hostname}}.pid"
   tags:

roles/odfees/templates/config/jvm.options.j2

@@ -19,8 +19,8 @@
 # Xms represents the initial size of total heap space
 # Xmx represents the maximum size of total heap space
 
--Xms512m
--Xmx512m
+-Xms{{odfe_javamem}}
+-Xmx{{odfe_javamem}}
 
 ################################################################
 ## Expert settings