Skip to content
Snippets Groups Projects
Commit 30010f63 authored by Arne Øslebø's avatar Arne Øslebø
Browse files

fixed problem with nifi pipeline

parent 5320b3ae
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
roles/nifi/tasks/main.yml
+ 56
− 4
View file @ 30010f63
......@@ -130,17 +130,69 @@
tags:
- start
- name: Download umbrella-top-1m.csv
get_url:
- name: Download umbrella-top-1m.csv.zip
local_action:
module: get_url
url: http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip
dest: "/tmp/umbrella-top-1m.csv.zip"
run_once: True
tags:
- start
- name: Unzip umbrella-top-1m.csv.zip
local_action:
module: unarchive
src: "/tmp/umbrella-top-1m.csv.zip"
dest: "/tmp"
run_once: True
tags:
- start
- name: Copy umbrella-top-1m.csv
copy:
src: "/tmp/top-1m.csv"
dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
tags:
- start
- name: Add header to umbrella-top-1m.csv
lineinfile:
path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/umbrella-top-1m.csv"
line: 'index,domain'
insertbefore: BOF
tags:
- start
- name: Download alexa-top-1m.csv
get_url:
- name: Download alexa-top-1m.csv.zip
local_action:
module: get_url
url: http://s3.amazonaws.com/alexa-static/top-1m.csv.zip
dest: "/tmp/alexa-top-1m.csv.zip"
run_once: True
tags:
- start
- name: Unzip alexa-top-1m.csv.zip
local_action:
module: unarchive
src: "/tmp/alexa-top-1m.csv.zip"
dest: "/tmp"
run_once: True
tags:
- start
- name: Copy alexa-top-1m.csv
copy:
src: "/tmp/top-1m.csv"
dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
tags:
- start
- name: Add header to alexa-top-1m.csv
lineinfile:
path: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/enrich/alexa-top-1m.csv"
line: 'index,domain'
insertbefore: BOF
tags:
- start
......
roles/nifi/templates/flow.xml.j2
+ 22
− 42
View file @ 30010f63
......@@ -4033,7 +4033,7 @@
<enabled>true</enabled>
<property>
<name>csv-file</name>
<value>/opt/nifi/nifi-current/conf/alexa-top-1m.csv</value>
<value>/opt/nifi/nifi-current/conf/enrich/alexa-top-1m.csv</value>
</property>
<property>
<name>CSV Format</name>
......@@ -4069,7 +4069,7 @@
<enabled>true</enabled>
<property>
<name>csv-file</name>
<value>/opt/nifi/nifi-current/conf/umbrella-top-1m.csv</value>
<value>/opt/nifi/nifi-current/conf/enrich/umbrella-top-1m.csv</value>
</property>
<property>
<name>CSV Format</name>
......@@ -4110,7 +4110,7 @@
<name>To data output</name>
<position x="-1120.0" y="592.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
</outputPort>
<processGroup>
<id>27d51d04-0172-1000-0000-00004573c6ec</id>
......@@ -4155,7 +4155,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
......@@ -4202,7 +4202,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
......@@ -4247,7 +4247,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
......@@ -4282,7 +4282,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
......@@ -4339,7 +4339,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
......@@ -4396,7 +4396,7 @@
<yieldPeriod>1 sec</yieldPeriod>
<bulletinLevel>WARN</bulletinLevel>
<lossTolerant>false</lossTolerant>
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
......@@ -4440,7 +4440,7 @@
<name>To storage</name>
<position x="-2120.0" y="1064.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
<scheduledState>STOPPED</scheduledState>
</outputPort>
<funnel>
<id>beabd3be-6f95-3369-9aa5-4631e6207ec5</id>
......@@ -4933,7 +4933,7 @@
</property>
<property>
<name>Port</name>
<value>5000</value>
<value>6000</value>
</property>
<property>
<name>Receive Buffer Size</name>
......@@ -4968,7 +4968,7 @@
</property>
<property>
<name>Client Auth</name>
<value>REQUIRED</value>
<value>NONE</value>
</property>
</processor>
<processor>
......@@ -5014,14 +5014,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<funnel>
<id>895faa7a-0175-1000-0000-000014ef9dd3</id>
<position x="278.84829417593915" y="332.4492766741185" />
</funnel>
<funnel>
<id>895f7db3-0175-1000-ffff-ffff8229d688</id>
<position x="-1446.1517058240609" y="301.4492766741185" />
</funnel>
<funnel>
<id>895faa7a-0175-1000-0000-000014ef9dd3</id>
<position x="278.84829417593915" y="332.4492766741185" />
</funnel>
<connection>
<id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id>
<name />
......@@ -5430,14 +5430,14 @@
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<funnel>
<id>d8f19295-5666-31a8-b701-52214c4db51d</id>
<position x="-1500.995244929405" y="257.20806784146276" />
</funnel>
<funnel>
<id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id>
<position x="283.72871497338747" y="273.4623850295515" />
</funnel>
<funnel>
<id>d8f19295-5666-31a8-b701-52214c4db51d</id>
<position x="-1500.995244929405" y="257.20806784146276" />
</funnel>
<processGroup>
<id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id>
<name>TLS events</name>
......@@ -7191,26 +7191,6 @@
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>214d5013-0175-1000-ffff-ffff9b7dbebb</id>
<name />
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>1ef39440-1985-3bbb-8e03-859a1c5ee4b1</sourceId>
<sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>21a9e277-2d80-359a-9c57-cb76d8962e6d</destinationId>
<destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>27d64272-0172-1000-0000-000079e1c9c6</id>
<name />
......@@ -7398,7 +7378,7 @@
</property>
<property>
<name>Password</name>
<value>enc{d3aa1dd95c404935f1bc2e54eefadb419acb135aaafa6edfd6881023c2a158c3ab4b958d1b765d4b8bc0a669b82fac3a}</value>
<value>enc{122e3a53e4a8959bb7f1b39e916fea2572e0d29014996e4288541e0fa8da0a4d65dad32242fc1817d2f07c967759113a}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
......@@ -10238,7 +10218,7 @@
</property>
<property>
<name>Truststore Password</name>
<value>enc{8aabd8fc1ac036b72d7f27f5e07e82c50119a1c5f61a705d6788218b4b769aa3}</value>
<value>enc{403bd69a9cb1b362f014b238fd0ad8dbf490d31810e3e5acb58ba40b4ba4b790}</value>
</property>
<property>
<name>Truststore Type</name>
......
roles/soctools-server/tasks/main.yml
+ 1
− 1
View file @ 30010f63
......@@ -16,7 +16,7 @@
- name: Install required packages
yum:
name: ["docker-ce","docker-ce-cli","containerd.io","python-pip"]
name: ["docker-ce","docker-ce-cli","containerd.io","python-pip","unzip"]
state: latest
validate_certs: no
when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux'
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment