stop, restart and update-config scripts for nifi

roles/build/templates/nifi/Dockerfile.j2

@@ -97,7 +97,7 @@ WORKDIR ${NIFI_HOME}
 # thus normal shell processing does not happen:
 # https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example
 USER root
-RUN yum install -y supervisor
+RUN yum install -y supervisor rsync
 RUN yum clean all
 COPY nifisupervisord.conf /etc/supervisord.conf
 ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

roles/nifi/tasks/main.yml

 ---
 
-- name: Copy cacert to ca-trust dir
-  remote_user: root
-  copy:
-    src: "{{playbook_dir}}/secrets/CA/ca.crt"
-    dest: /etc/pki/ca-trust/source/anchors/ca.crt
-  tags:
-    - start
-
-- name: Install cacert to root truststore
-  remote_user: root
-  command: "update-ca-trust"
-  tags:
-    - start
-
-- name: Copy certificates in NiFi conf dir
-  remote_user: nifi
-  copy:
-    src:  "{{ item }}"
-    dest: "conf/"
-  with_items:
-    - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12"
-    - "{{playbook_dir}}/secrets/CA/cacerts.jks"
-    - common-cacerts.jks
-  tags:
-    - start
-
-- name: Configure flow.xml
-  remote_user: nifi
-  template:
-    src:  "flow.xml.j2"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
-  tags:
-    - start
-
-- name: Gzip flow.xml
-  remote_user: nifi
-  archive:
-    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
-    format: gz
-  tags:
-    - start
-
-- name: Get openid authkey
-  remote_user: nifi
-  set_fact:
-    nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}"
-  tags:
-    - start
-
-- name: Configure NiFi boostrap properties
-  remote_user: nifi
-  template:
-    src: bootstrap.conf.j2
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf"
-  tags:
-    - start
-
-- name: Configure NiFi properties for secure servers
-  remote_user: nifi
-  template:
-    src: nifi.properties.j2
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties"
-  tags:
-    - start
-
-- name: Copy authorizations.xml
-  remote_user: nifi
-  copy:
-    src: "authorizations.xml"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml"
-  tags:
-    - start
-
-- name: Configure users
-  remote_user: nifi
-  template:
-    src: users.xml.j2
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml"
-  tags:
-    - start
-
-- name: Configure NiFi authorizers for secure servers
-  remote_user: nifi
-  template:
-    src: authorizers.xml.j2
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml"
-  tags:
-    - start
-
-- name: Create conf/enrich dir
-  remote_user: nifi
-  file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory
-  tags:
-   - start
-
-- name: Create conf/enrich/freq dir
-  remote_user: nifi
-  file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory
-  tags:
-   - start
-
-- name: Download freq processor
-  remote_user: nifi
-  get_url:
-    url: 'https://scm.uninett.no/geant-wp8-t3.1/nifi-processors/-/raw/master/scripts/freq/{{ item }}'
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/"
-  with_items:
-   - alexa.json
-   - freq.py
-   - freqProcessor.py
-  tags:
-   - start
-
-- name: Copy empty GeoLite2-City database
-  remote_user: nifi
-  copy:
-    src: GeoLite2-City.mmdb
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb"
-  tags:
-    - start
-
-- name: Copy CountriesWithRegionalCodes.csv
-  remote_user: nifi
-  copy:
-    src: CountriesWithRegionalCodes.csv
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv"
-  tags:
-    - start
-
-- name: Copy grok libraries
-  copy:
-   src: haproxy.groklib
-   dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib"
+- include: start.yml
   tags:
    - start
-
-- name: Create empty list of Tor nodes
-  remote_user: nifi
-  copy:
-    content: "ip_addr,value"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv"
-    force: no
+- include: stop.yml
   tags:
-   - start
-
-- name: Download umbrella-top-1m.csv.zip
-  remote_user: nifi
-  local_action:
-    module: get_url
-    url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip
-    dest: "/tmp/umbrella-top-1m.csv.zip"
-  run_once: True
+   - stop
+   - stop-nifi
+- include: update-config.yml
   tags:
-   - start
-
-- name: Unzip umbrella-top-1m.csv.zip
-  remote_user: nifi
-  local_action:
-    module: unarchive
-    src: "/tmp/umbrella-top-1m.csv.zip"
-    dest: "/tmp"
-  run_once: True
+   - update-config
+   - update-nifi-config
+- include: restart.yml
   tags:
-   - start
-
-- name: Copy umbrella-top-1m.csv
-  remote_user: nifi
-  copy:
-    src: "/tmp/top-1m.csv"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
-  tags:
-   - start 
-
-- name: Add header to umbrella-top-1m.csv
-  remote_user: nifi
-  lineinfile:
-    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
-    line: 'index,domain'
-    insertbefore: BOF
-  tags:
-   - start
-
-- name: Download alexa-top-1m.csv.zip
-  remote_user: nifi
-  local_action:
-    module: get_url
-    url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip
-    dest: "/tmp/alexa-top-1m.csv.zip"
-  run_once: True
-  tags:
-   - start
-
-- name: Unzip alexa-top-1m.csv.zip
-  remote_user: nifi
-  local_action:
-    module: unarchive
-    src: "/tmp/alexa-top-1m.csv.zip"
-    dest: "/tmp"
-  run_once: True
-  tags:
-   - start
-
-- name: Copy alexa-top-1m.csv
-  remote_user: nifi
-  copy:
-    src: "/tmp/top-1m.csv"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
-  tags:
-   - start 
-
-- name: Add header to alexa-top-1m.csv
-  remote_user: nifi
-  lineinfile:
-    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
-    line: 'index,domain'
-    insertbefore: BOF
-  tags:
-   - start
-
-- name: Start NiFi
-  remote_user: root
-  command: "supervisorctl start nifi"
-  tags:
-    - start
-
-#- name: check reachable hosts
-#  gather_facts: no
-#  tasks:
-#    - command: ping -c1 {{ inventory_hostname }}
-#      delegate_to: localhost
-#      register: ping_result
-#      ignore_errors: yes
-#    - group_by: key=reachable
-#      when: ping_result|success
-
-- name: Set Autostart for supervisord's services
-  shell: "sed -i 's/autostart=false/autostart=true/g' /etc/supervisord.conf"
-  tags:
-    - start
-
-- name: Stop NiFi
-  remote_user: root
-  command: "supervisorctl stop nifi"
-  tags:
-    - stop
-
-#- name: Copy flow from NiFi
-#  fetch:
-#    src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
-#    dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
-#    flat: yes
-#  tags:
-#    - stop
-
+   - restart
+   - restart-nifi

roles/nifi/tasks/restart.yml

+---
+
+- name: Restart NiFi
+  remote_user: root
+  command: "supervisorctl restart nifi"
+

roles/nifi/tasks/start.yml

+---
+
+- name: Copy cacert to ca-trust dir
+  remote_user: root
+  copy:
+    src: "{{playbook_dir}}/secrets/CA/ca.crt"
+    dest: /etc/pki/ca-trust/source/anchors/ca.crt
+
+- name: Install cacert to root truststore
+  remote_user: root
+  command: "update-ca-trust"
+
+- name: Copy certificates in NiFi conf dir
+  remote_user: nifi
+  copy:
+    src:  "{{ item }}"
+    dest: "conf/"
+  with_items:
+    - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12"
+    - "{{playbook_dir}}/secrets/CA/cacerts.jks"
+    - common-cacerts.jks
+
+- name: Configure flow.xml
+  remote_user: nifi
+  template:
+    src:  "flow.xml.j2"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+
+- name: Gzip flow.xml
+  remote_user: nifi
+  archive:
+    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+    format: gz
+
+- name: Get openid authkey
+  remote_user: nifi
+  set_fact:
+    nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}"
+
+- name: Configure NiFi boostrap properties
+  remote_user: nifi
+  template:
+    src: bootstrap.conf.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf"
+
+- name: Configure NiFi properties for secure servers
+  remote_user: nifi
+  template:
+    src: nifi.properties.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties"
+
+- name: Copy authorizations.xml
+  remote_user: nifi
+  copy:
+    src: "authorizations.xml"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml"
+
+- name: Configure users
+  remote_user: nifi
+  template:
+    src: users.xml.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml"
+
+- name: Configure NiFi authorizers for secure servers
+  remote_user: nifi
+  template:
+    src: authorizers.xml.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml"
+
+- name: Create conf/enrich dir
+  remote_user: nifi
+  file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory
+
+- name: Create conf/enrich/freq dir
+  remote_user: nifi
+  file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory
+
+- name: Download freq processor
+  remote_user: nifi
+  get_url:
+    url: 'https://scm.uninett.no/geant-wp8-t3.1/nifi-processors/-/raw/master/scripts/freq/{{ item }}'
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/"
+  with_items:
+   - alexa.json
+   - freq.py
+   - freqProcessor.py
+
+- name: Copy empty GeoLite2-City database
+  remote_user: nifi
+  copy:
+    src: GeoLite2-City.mmdb
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb"
+
+- name: Copy CountriesWithRegionalCodes.csv
+  remote_user: nifi
+  copy:
+    src: CountriesWithRegionalCodes.csv
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv"
+
+- name: Copy grok libraries
+  copy:
+   src: haproxy.groklib
+   dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib"
+
+- name: Create empty list of Tor nodes
+  remote_user: nifi
+  copy:
+    content: "ip_addr,value"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv"
+    force: no
+
+- name: Download umbrella-top-1m.csv.zip
+  remote_user: nifi
+  local_action:
+    module: get_url
+    url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip
+    dest: "/tmp/umbrella-top-1m.csv.zip"
+  run_once: True
+
+- name: Unzip umbrella-top-1m.csv.zip
+  remote_user: nifi
+  local_action:
+    module: unarchive
+    src: "/tmp/umbrella-top-1m.csv.zip"
+    dest: "/tmp"
+  run_once: True
+
+- name: Copy umbrella-top-1m.csv
+  remote_user: nifi
+  copy:
+    src: "/tmp/top-1m.csv"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
+
+- name: Add header to umbrella-top-1m.csv
+  remote_user: nifi
+  lineinfile:
+    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
+    line: 'index,domain'
+    insertbefore: BOF
+
+- name: Download alexa-top-1m.csv.zip
+  remote_user: nifi
+  local_action:
+    module: get_url
+    url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip
+    dest: "/tmp/alexa-top-1m.csv.zip"
+  run_once: True
+
+- name: Unzip alexa-top-1m.csv.zip
+  remote_user: nifi
+  local_action:
+    module: unarchive
+    src: "/tmp/alexa-top-1m.csv.zip"
+    dest: "/tmp"
+  run_once: True
+
+- name: Copy alexa-top-1m.csv
+  remote_user: nifi
+  copy:
+    src: "/tmp/top-1m.csv"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
+
+- name: Add header to alexa-top-1m.csv
+  remote_user: nifi
+  lineinfile:
+    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
+    line: 'index,domain'
+    insertbefore: BOF
+
+- name: Start NiFi
+  remote_user: root
+  command: "supervisorctl start nifi"
+
+#- name: check reachable hosts
+#  gather_facts: no
+#  tasks:
+#    - command: ping -c1 {{ inventory_hostname }}
+#      delegate_to: localhost
+#      register: ping_result
+#      ignore_errors: yes
+#    - group_by: key=reachable
+#      when: ping_result|success
+
+- name: Set Autostart for supervisord's services
+  replace:
+    path: /etc/supervisord.conf
+    regexp: '^autostart=false$'
+    replace: 'autostart=true'
+
+#- name: Copy flow from NiFi
+#  fetch:
+#    src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+#    dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
+#    flat: yes
+

roles/nifi/tasks/stop.yml

+---
+
+- name: Stop NiFi
+  remote_user: root
+  command: "supervisorctl stop nifi"
+

roles/nifi/tasks/update-config.yml

+---
+
+- name: Configure flow.xml
+  remote_user: nifi
+  template:
+    src:  "flow.xml.j2"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+
+- name: Gzip flow.xml
+  remote_user: nifi
+  archive:
+    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+    format: gz
+
+- name: Configure NiFi boostrap properties
+  remote_user: nifi
+  template:
+    src: bootstrap.conf.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf"
+
+- name: Configure NiFi properties for secure servers
+  remote_user: nifi
+  template:
+    src: nifi.properties.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties"
+
+- name: Copy authorizations.xml
+  remote_user: nifi
+  copy:
+    src: "authorizations.xml"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml"
+
+- name: Configure users
+  remote_user: nifi
+  template:
+    src: users.xml.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml"
+
+- name: Configure NiFi authorizers for secure servers
+  remote_user: nifi
+  template:
+    src: authorizers.xml.j2
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml"
+
+- name: Copy empty GeoLite2-City database
+  remote_user: nifi
+  copy:
+    src: GeoLite2-City.mmdb
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb"
+
+- name: Copy CountriesWithRegionalCodes.csv
+  remote_user: nifi
+  copy:
+    src: CountriesWithRegionalCodes.csv
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv"
+
+- name: Copy grok libraries
+  copy:
+   src: haproxy.groklib
+   dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib"
+
+- name: Create empty list of Tor nodes
+  remote_user: nifi
+  copy:
+    content: "ip_addr,value"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv"
+    force: no
+