added support for automatic configuration of Nifi pipeline and policies

group_vars/all/main.yml

@@ -18,7 +18,7 @@ nifi_img: "{{repo}}/nifi:{{version}}{{suffix}}"
 nginx_name: "dsoclab-nginx"
 nginx_img: "{{repo}}/nginx:{{version}}{{suffix}}"
 
-dslproxy: "dsoclab.gn4-3-wp8-soc.sunet.se"
+dslproxy: "arne-centos.cert-labs.uninett.no"
 
 kspass: "Testing003"
 tspass: "Testing003"
@@ -37,13 +37,6 @@ nifi_repo: "https://archive.apache.org/dist"
 ca_cn: "SOCTOOLS-CA"
 
 soctools_users:
-  - firstname: "Bozidar"
-    lastname: "Proevski"
-    username: "bozidar.proevski"
-    email: "bozidar.proevski@finki.ukim.mk"
-    DN: "CN=Bozidar Proevski"
-    CN: "Bozidar Proevski"
-    password: "Pass001"
   - firstname: "Arne"
     lastname: "Oslebo"
     username: "arne.oslebo"
@@ -51,7 +44,13 @@ soctools_users:
     DN: "CN=Arne Oslebo"
     CN: "Arne Oslebo"
     password: "Pass002"
-
+  - firstname: "Bozidar"
+    lastname: "Proevski"
+    username: "bozidar.proevski"
+    email: "bozidar.proevski@finki.ukim.mk"
+    DN: "CN=Bozidar Proevski"
+    CN: "Bozidar Proevski"
+    password: "Pass001"
 
 odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}"
 odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}"
@@ -69,3 +68,9 @@ openid_subjkey: preferred_username
 
 keycloak_img: "{{repo}}/keycloak:{{version}}{{suffix}}"
 keycloak_adminpass: "Pass005"
+
+elastic_url: "https://dsoclab-odfe-1:9200"
+elastic_username: "admin"
+misp_token: ""
+misp_url: ""
+maxmind_key: ""

roles/nifi/files/authorizations.xml

+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<authorizations>
+    <policies>
+        <policy identifier="f99bccd1-a30e-3e4a-98a2-dbc708edc67f" resource="/flow" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="b8775bd4-704a-34c6-987b-84f2daf7a515" resource="/restricted-components" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="627410be-1717-35b4-a06f-e9362b89e0b7" resource="/tenants" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="15e4e0bd-cb28-34fd-8587-f8d15162cba5" resource="/tenants" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="ff96062a-fa99-36dc-9942-0f6442ae7212" resource="/policies" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="ad99ea98-3af6-3561-ae27-5bf09e1d969d" resource="/policies" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="2e1015cb-0fed-3005-8e0d-722311f21a03" resource="/controller" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="c6322e6c-4cc1-3bcc-91b3-2ed2111674cf" resource="/controller" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="287edf48-da72-359b-8f61-da5d4c45a270" resource="/proxy" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000001"/>
+        </policy>
+        <policy identifier="cbb1ca56-0172-1000-0000-000054f59541" resource="/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb46de3-0172-1000-0000-0000012e34b0" resource="/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb49910-0172-1000-ffff-fffffe53fc76" resource="/operation/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb4ce71-0172-1000-ffff-ffffc8beda52" resource="/provenance-data/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb4f792-0172-1000-ffff-ffffaf98b8d6" resource="/data/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000001"/>
+	    <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb52642-0172-1000-ffff-ffff926f1c64" resource="/data/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000001"/>
+	    <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb55fb7-0172-1000-0000-000012a22cb2" resource="/policies/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="R">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+        <policy identifier="cbb581b0-0172-1000-ffff-ffffbbf07efe" resource="/policies/process-groups/8652e374-0170-1000-4012-2e89251a60f9" action="W">
+            <group identifier="c78caf19-016f-1000-0000-000000000002"/>
+        </policy>
+    </policies>
+</authorizations>

roles/nifi/tasks/main.yml

@@ -24,15 +24,19 @@
   tags:
     - start
 
-- name: Copy flow in NiFi conf dir
-  copy:
-    src:  "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
-    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
-  when:
-    - "'{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz' is is_file"
+- name: Configure flow.xml
+  template:
+    src:  "flow.xml.j2"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
   tags:
     - start
 
+- name: Gzip flow.xml
+  archive:
+    path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+    format: gz
+
 - name: Get openid authkey
   set_fact:
     nifisecret: "{{lookup('file', 'files/nifisecret',convert_data=False) | from_json }}"
@@ -53,6 +57,20 @@
   tags:
     - start
 
+- name: Copy authorizations.xml
+  copy:
+    src: "authorizations.xml"
+    dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/authorizations.xml"
+  tags:
+    - start
+
+- name: Configure users
+  template:
+    src: users.xml.j2
+    dest: conf/users.xml
+  tags:
+    - start
+
 - name: Configure NiFi authorizers for secure servers
   template:
     src: authorizers.xml.j2
@@ -60,6 +78,22 @@
   tags:
     - start
 
+- name: Copy empty GeoLite2-City database
+  copy:
+    src: GeoLite2-City.mmdb
+    dest: conf/enrich/
+  tags:
+    - start
+
+- name: Create empty enrichment files
+  copy:
+    content: ""
+    dest: conf/enrich/{{ item }}
+  with_items:
+    - "tornodes.csv"
+    - "umbrella-top-1m.csv"
+    - "alexa-top-1m.csv"
+
 - name: Start NiFi
   command: "./bin/nifi.sh start"
   tags:
@@ -80,11 +114,11 @@
   tags:
     - stop
 
-- name: Copy flow from NiFi
-  fetch:
-    src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
-    dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
-    flat: yes
-  tags:
-    - stop
+#- name: Copy flow from NiFi
+#  fetch:
+#    src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
+#    dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
+#    flat: yes
+#  tags:
+#    - stop
 

roles/nifi/templates/authorizers.xml.j2

@@ -49,7 +49,9 @@
         <property name="Users File">./conf/users.xml</property>
         <property name="Legacy Authorized Users File"></property>
 
-        <property name="Initial User Identity 1">{{soctools_users[0].username}}</property>
+{% for user in soctools_users %}
+        <property name="Initial User Identity 1">{{user.username}}</property>
+{% endfor %}        
 {% for nifi in groups['nificontainers'] %}
         <property name="Initial User Identity {{ loop.index +1 }}">CN={{ nifi }}</property>
 {% endfor %}

roles/nifi/templates/users.xml.j2

+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<tenants>
+    <groups>
+      <group identifier="c78caf19-016f-1000-0000-000000000001" name="NiFi nodes">
+	{% for nifi in groups['nificontainers'] %}
+	<user identifier="c78caf19-016f-1000-0001-{{'%012d'|format(loop.index) }}"/>
+	{% endfor %}
+      </group>
+      <group identifier="c78caf19-016f-1000-0000-000000000002" name="Administrators">
+	{% for user in soctools_users %}
+	<user identifier="c78caf19-016f-1000-0002-{{'%012d'|format(loop.index) }}"/>
+	{% endfor %}
+      </group>
+    </groups>
+    <users>
+	{% for nifi in groups['nificontainers'] %}
+        <user identifier="c78caf19-016f-1000-0001-{{'%012d'|format(loop.index) }}" identity="CN={{ nifi }}"/>
+        {% endfor %}
+	{% for user in soctools_users %}
+        <user identifier="c78caf19-016f-1000-0002-{{'%012d'|format(loop.index) }}" identity="{{ user.username }}"/>
+        {% endfor %}        
+    </users>
+</tenants>