diff --git a/roles/build/templates/nifi/Dockerfile.j2 b/roles/build/templates/nifi/Dockerfile.j2 index 916c96c42771ba4a76a7c767a8521566862041c1..63c51747ea349daa1509f556ee4c323043635c3c 100644 --- a/roles/build/templates/nifi/Dockerfile.j2 +++ b/roles/build/templates/nifi/Dockerfile.j2 @@ -97,7 +97,7 @@ WORKDIR ${NIFI_HOME} # thus normal shell processing does not happen: # https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example USER root -RUN yum install -y supervisor +RUN yum install -y supervisor rsync RUN yum clean all COPY nifisupervisord.conf /etc/supervisord.conf ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] diff --git a/roles/nifi/tasks/main.yml b/roles/nifi/tasks/main.yml index d40885d26a03128c02cea825800bd269730ac50f..d4ab3ad3a89413bf3d6a848822562a5ccb6d92ca 100644 --- a/roles/nifi/tasks/main.yml +++ b/roles/nifi/tasks/main.yml @@ -1,257 +1,17 @@ --- -- name: Copy cacert to ca-trust dir - remote_user: root - copy: - src: "{{playbook_dir}}/secrets/CA/ca.crt" - dest: /etc/pki/ca-trust/source/anchors/ca.crt - tags: - - start - -- name: Install cacert to root truststore - remote_user: root - command: "update-ca-trust" - tags: - - start - -- name: Copy certificates in NiFi conf dir - remote_user: nifi - copy: - src: "{{ item }}" - dest: "conf/" - with_items: - - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" - - "{{playbook_dir}}/secrets/CA/cacerts.jks" - - common-cacerts.jks - tags: - - start - -- name: Configure flow.xml - remote_user: nifi - template: - src: "flow.xml.j2" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" - tags: - - start - -- name: Gzip flow.xml - remote_user: nifi - archive: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" - format: gz - tags: - - start - -- name: Get openid authkey - remote_user: nifi - set_fact: - nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}" - tags: - - start - -- name: Configure NiFi boostrap properties - remote_user: nifi - template: - src: bootstrap.conf.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf" - tags: - - start - -- name: Configure NiFi properties for secure servers - remote_user: nifi - template: - src: nifi.properties.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties" - tags: - - start - -- name: Copy authorizations.xml - remote_user: nifi - copy: - src: "authorizations.xml" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml" - tags: - - start - -- name: Configure users - remote_user: nifi - template: - src: users.xml.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml" - tags: - - start - -- name: Configure NiFi authorizers for secure servers - remote_user: nifi - template: - src: authorizers.xml.j2 - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml" - tags: - - start - -- name: Create conf/enrich dir - remote_user: nifi - file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory - tags: - - start - -- name: Create conf/enrich/freq dir - remote_user: nifi - file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory - tags: - - start - -- name: Download freq processor - remote_user: nifi - get_url: - url: 'https://scm.uninett.no/geant-wp8-t3.1/nifi-processors/-/raw/master/scripts/freq/{{ item }}' - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/" - with_items: - - alexa.json - - freq.py - - freqProcessor.py - tags: - - start - -- name: Copy empty GeoLite2-City database - remote_user: nifi - copy: - src: GeoLite2-City.mmdb - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb" - tags: - - start - -- name: Copy CountriesWithRegionalCodes.csv - remote_user: nifi - copy: - src: CountriesWithRegionalCodes.csv - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv" - tags: - - start - -- name: Copy grok libraries - copy: - src: haproxy.groklib - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib" +- include: start.yml tags: - start - -- name: Create empty list of Tor nodes - remote_user: nifi - copy: - content: "ip_addr,value" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv" - force: no +- include: stop.yml tags: - - start - -- name: Download umbrella-top-1m.csv.zip - remote_user: nifi - local_action: - module: get_url - url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip - dest: "/tmp/umbrella-top-1m.csv.zip" - run_once: True + - stop + - stop-nifi +- include: update-config.yml tags: - - start - -- name: Unzip umbrella-top-1m.csv.zip - remote_user: nifi - local_action: - module: unarchive - src: "/tmp/umbrella-top-1m.csv.zip" - dest: "/tmp" - run_once: True + - update-config + - update-nifi-config +- include: restart.yml tags: - - start - -- name: Copy umbrella-top-1m.csv - remote_user: nifi - copy: - src: "/tmp/top-1m.csv" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" - tags: - - start - -- name: Add header to umbrella-top-1m.csv - remote_user: nifi - lineinfile: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" - line: 'index,domain' - insertbefore: BOF - tags: - - start - -- name: Download alexa-top-1m.csv.zip - remote_user: nifi - local_action: - module: get_url - url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip - dest: "/tmp/alexa-top-1m.csv.zip" - run_once: True - tags: - - start - -- name: Unzip alexa-top-1m.csv.zip - remote_user: nifi - local_action: - module: unarchive - src: "/tmp/alexa-top-1m.csv.zip" - dest: "/tmp" - run_once: True - tags: - - start - -- name: Copy alexa-top-1m.csv - remote_user: nifi - copy: - src: "/tmp/top-1m.csv" - dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" - tags: - - start - -- name: Add header to alexa-top-1m.csv - remote_user: nifi - lineinfile: - path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" - line: 'index,domain' - insertbefore: BOF - tags: - - start - -- name: Start NiFi - remote_user: root - command: "supervisorctl start nifi" - tags: - - start - -#- name: check reachable hosts -# gather_facts: no -# tasks: -# - command: ping -c1 {{ inventory_hostname }} -# delegate_to: localhost -# register: ping_result -# ignore_errors: yes -# - group_by: key=reachable -# when: ping_result|success - -- name: Set Autostart for supervisord's services - shell: "sed -i 's/autostart=false/autostart=true/g' /etc/supervisord.conf" - tags: - - start - -- name: Stop NiFi - remote_user: root - command: "supervisorctl stop nifi" - tags: - - stop - -#- name: Copy flow from NiFi -# fetch: -# src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" -# dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz" -# flat: yes -# tags: -# - stop - + - restart + - restart-nifi diff --git a/roles/nifi/tasks/restart.yml b/roles/nifi/tasks/restart.yml new file mode 100644 index 0000000000000000000000000000000000000000..d5434f2a368c8bfa115e96c39708d4c6764438b3 --- /dev/null +++ b/roles/nifi/tasks/restart.yml @@ -0,0 +1,6 @@ +--- + +- name: Restart NiFi + remote_user: root + command: "supervisorctl restart nifi" + diff --git a/roles/nifi/tasks/start.yml b/roles/nifi/tasks/start.yml new file mode 100644 index 0000000000000000000000000000000000000000..901ca082e29f096ab6be6e0677b804af7166f9b0 --- /dev/null +++ b/roles/nifi/tasks/start.yml @@ -0,0 +1,196 @@ +--- + +- name: Copy cacert to ca-trust dir + remote_user: root + copy: + src: "{{playbook_dir}}/secrets/CA/ca.crt" + dest: /etc/pki/ca-trust/source/anchors/ca.crt + +- name: Install cacert to root truststore + remote_user: root + command: "update-ca-trust" + +- name: Copy certificates in NiFi conf dir + remote_user: nifi + copy: + src: "{{ item }}" + dest: "conf/" + with_items: + - "{{playbook_dir}}/secrets/CA/private/{{ inventory_hostname }}.p12" + - "{{playbook_dir}}/secrets/CA/cacerts.jks" + - common-cacerts.jks + +- name: Configure flow.xml + remote_user: nifi + template: + src: "flow.xml.j2" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + +- name: Gzip flow.xml + remote_user: nifi + archive: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" + format: gz + +- name: Get openid authkey + remote_user: nifi + set_fact: + nifisecret: "{{lookup('file', '{{playbook_dir}}/secrets/tokens/nifisecret',convert_data=False)}}" + +- name: Configure NiFi boostrap properties + remote_user: nifi + template: + src: bootstrap.conf.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf" + +- name: Configure NiFi properties for secure servers + remote_user: nifi + template: + src: nifi.properties.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties" + +- name: Copy authorizations.xml + remote_user: nifi + copy: + src: "authorizations.xml" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml" + +- name: Configure users + remote_user: nifi + template: + src: users.xml.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml" + +- name: Configure NiFi authorizers for secure servers + remote_user: nifi + template: + src: authorizers.xml.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml" + +- name: Create conf/enrich dir + remote_user: nifi + file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich state=directory + +- name: Create conf/enrich/freq dir + remote_user: nifi + file: path={{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq state=directory + +- name: Download freq processor + remote_user: nifi + get_url: + url: 'https://scm.uninett.no/geant-wp8-t3.1/nifi-processors/-/raw/master/scripts/freq/{{ item }}' + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/freq/" + with_items: + - alexa.json + - freq.py + - freqProcessor.py + +- name: Copy empty GeoLite2-City database + remote_user: nifi + copy: + src: GeoLite2-City.mmdb + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb" + +- name: Copy CountriesWithRegionalCodes.csv + remote_user: nifi + copy: + src: CountriesWithRegionalCodes.csv + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv" + +- name: Copy grok libraries + copy: + src: haproxy.groklib + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib" + +- name: Create empty list of Tor nodes + remote_user: nifi + copy: + content: "ip_addr,value" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv" + force: no + +- name: Download umbrella-top-1m.csv.zip + remote_user: nifi + local_action: + module: get_url + url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip + dest: "/tmp/umbrella-top-1m.csv.zip" + run_once: True + +- name: Unzip umbrella-top-1m.csv.zip + remote_user: nifi + local_action: + module: unarchive + src: "/tmp/umbrella-top-1m.csv.zip" + dest: "/tmp" + run_once: True + +- name: Copy umbrella-top-1m.csv + remote_user: nifi + copy: + src: "/tmp/top-1m.csv" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" + +- name: Add header to umbrella-top-1m.csv + remote_user: nifi + lineinfile: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv" + line: 'index,domain' + insertbefore: BOF + +- name: Download alexa-top-1m.csv.zip + remote_user: nifi + local_action: + module: get_url + url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip + dest: "/tmp/alexa-top-1m.csv.zip" + run_once: True + +- name: Unzip alexa-top-1m.csv.zip + remote_user: nifi + local_action: + module: unarchive + src: "/tmp/alexa-top-1m.csv.zip" + dest: "/tmp" + run_once: True + +- name: Copy alexa-top-1m.csv + remote_user: nifi + copy: + src: "/tmp/top-1m.csv" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" + +- name: Add header to alexa-top-1m.csv + remote_user: nifi + lineinfile: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv" + line: 'index,domain' + insertbefore: BOF + +- name: Start NiFi + remote_user: root + command: "supervisorctl start nifi" + +#- name: check reachable hosts +# gather_facts: no +# tasks: +# - command: ping -c1 {{ inventory_hostname }} +# delegate_to: localhost +# register: ping_result +# ignore_errors: yes +# - group_by: key=reachable +# when: ping_result|success + +- name: Set Autostart for supervisord's services + replace: + path: /etc/supervisord.conf + regexp: '^autostart=false$' + replace: 'autostart=true' + +#- name: Copy flow from NiFi +# fetch: +# src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" +# dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz" +# flat: yes + diff --git a/roles/nifi/tasks/stop.yml b/roles/nifi/tasks/stop.yml new file mode 100644 index 0000000000000000000000000000000000000000..d6abd9a5bf0d555b0c3d531358c59b24a4044bbc --- /dev/null +++ b/roles/nifi/tasks/stop.yml @@ -0,0 +1,6 @@ +--- + +- name: Stop NiFi + remote_user: root + command: "supervisorctl stop nifi" + diff --git a/roles/nifi/tasks/update-config.yml b/roles/nifi/tasks/update-config.yml new file mode 100644 index 0000000000000000000000000000000000000000..a9ae6965d804b00ca6df70dff6813fb080bc5457 --- /dev/null +++ b/roles/nifi/tasks/update-config.yml @@ -0,0 +1,69 @@ +--- + +- name: Configure flow.xml + remote_user: nifi + template: + src: "flow.xml.j2" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + +- name: Gzip flow.xml + remote_user: nifi + archive: + path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz" + format: gz + +- name: Configure NiFi boostrap properties + remote_user: nifi + template: + src: bootstrap.conf.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/bootstrap.conf" + +- name: Configure NiFi properties for secure servers + remote_user: nifi + template: + src: nifi.properties.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/nifi.properties" + +- name: Copy authorizations.xml + remote_user: nifi + copy: + src: "authorizations.xml" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml" + +- name: Configure users + remote_user: nifi + template: + src: users.xml.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/users.xml" + +- name: Configure NiFi authorizers for secure servers + remote_user: nifi + template: + src: authorizers.xml.j2 + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizers.xml" + +- name: Copy empty GeoLite2-City database + remote_user: nifi + copy: + src: GeoLite2-City.mmdb + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/GeoLite2-City.mmdb" + +- name: Copy CountriesWithRegionalCodes.csv + remote_user: nifi + copy: + src: CountriesWithRegionalCodes.csv + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/CountriesWithRegionalCodes.csv" + +- name: Copy grok libraries + copy: + src: haproxy.groklib + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/haproxy.groklib" + +- name: Create empty list of Tor nodes + remote_user: nifi + copy: + content: "ip_addr,value" + dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/tornodes.csv" + force: no +