diff --git a/group_vars/all/main.yml b/group_vars/all/main.yml index 94834298e12b3747e1a87b7546915338fccb0da5..69e9f431b9a4ba768620dc1ac48c530d03166ad2 100644 --- a/group_vars/all/main.yml +++ b/group_vars/all/main.yml @@ -20,28 +20,11 @@ haproxy_img: "{{repo}}/haproxy:{{version}}{{suffix}}" HAPROXY_PROCESSES: "2" HAPROXY_STATS_PASS: "eiph2Eepaizicheelah3tei+bae3ohgh" -FILEBEAT_VERSION: "7.4.1" +FILEBEAT_VERSION: "7.9.3" FILEBEAT_OUTPUT_HOST: "{{dslproxy}}" -FILEBEAT_OUTPUT_PORT: "5019" +FILEBEAT_OUTPUT_PORT: "6000" FILEBEAT_CERT: "/opt/filebeat/filebeat.crt" FILEBEAT_KEY: "/opt/filebeat/filebeat.key" -FILEBEAT_NIFI_FILES: - - /var/log/*.log -FILEBEAT_HAPROXY_FILES: - - /var/log/*.log -FILEBEAT_KEYCLOAK_FILES: - - /var/log/*.log -FILEBEAT_MISP_FILES: - - /var/log/*.log -FILEBEAT_MYSQL_FILES: - - /var/log/*.log -FILEBEAT_ODFEES_FILES: - - /var/log/*.log -FILEBEAT_ODFEKIBANA_FILES: - - /var/log/*.log -FILEBEAT_ZOOKEEPER_FILES: - - /var/log/*.log - temp_root: "/tmp/centosbuild" @@ -63,14 +46,14 @@ kspass: "Testing003" tspass: "Testing003" sysctlconfig: - - { key: "net.core.rmem_max", val: "2097152" } - - { key: "net.core.wmem_max", val: "2097152" } + - { key: "net.core.rmem_max", val: "4194304" } + - { key: "net.core.wmem_max", val: "4194304" } - { key: "vm.max_map_count" , val: "524288" } nifi_javamem: "4g" odfe_javamem: "2g" -nifi_version: 1.11.4 +nifi_version: 1.12.1 nifi_repo: "https://archive.apache.org/dist" ca_cn: "SOCTOOLS-CA" diff --git a/roles/filebeat/templates/filebeat.yml.j2 b/roles/filebeat/templates/filebeat.yml.j2 index e86853fe3e822f7539cb998929acbcbdd0655ae0..3abc622e8a5a977305af304cd5c4ba108b0a01f2 100644 --- a/roles/filebeat/templates/filebeat.yml.j2 +++ b/roles/filebeat/templates/filebeat.yml.j2 @@ -4,30 +4,17 @@ filebeat.inputs: {% for file in FILEBEAT_FILES %} - {{ file }} {% endfor %} - json.keys_under_root: true - json.overwrite_keys: true - json.add_error_key: true - fields_under_root: true + fields: + log_type: {{ FILEBEAT_LOG_TYPE }} -setup.template.enabled: false -setup.ilm.enabled: false +filebeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false output.logstash: - enabled: true - protocol: https hosts: ["{{ FILEBEAT_OUTPUT_HOST }}:{{ FILEBEAT_OUTPUT_PORT }}"] - worker: 4 - compression_level: 0 + workers: 3 loadbalance: true - ssl: - enabled: false - certificate: "{{ FILEBEAT_CERT }}" - key: "{{ FILEBEAT_KEY }}" - -logging.level: info -processors: - - drop_fields: - fields: ["beat.hostname","beat.name","beat.version","offset","pid","v"] - - add_fields: - fields: - log_type: {{ FILEBEAT_LOG_TYPE }} + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + #ssl.certificate: "{{ FILEBEAT_CERT }}"" + #ssl.key: "{{ FILEBEAT_KEY }}" diff --git a/roles/nifi/templates/flow.xml.j2 b/roles/nifi/templates/flow.xml.j2 index f8605d1ca273e62a612d33d1ff9263f69e2bbe7d..787e171759191c7f7b96db24c43fb0ec9328a795 100644 --- a/roles/nifi/templates/flow.xml.j2 +++ b/roles/nifi/templates/flow.xml.j2 @@ -8,16 +8,22 @@ <name>NiFi Flow</name> <position x="0.0" y="0.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processGroup> <id>41088add-955b-3611-a0de-2c18b79b678c</id> <name>Data processing</name> <position x="1216.0" y="256.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processGroup> <id>fcbcacd1-542d-3a15-a5aa-9c1302328954</id> <name>Enrichment</name> <position x="384.0" y="720.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>01b66126-695a-3059-b179-f1bf85e8ca91</id> <name>Check fqdn enrichment</name> @@ -28,7 +34,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -59,7 +65,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -90,7 +96,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -130,6 +136,8 @@ <name>fqdn enrichment</name> <position x="-202.47354083453774" y="596.1945491887745" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>2703fe3e-7e6a-310e-a010-b30898befa9b</id> <name>Check fqdn2 enrichment</name> @@ -140,7 +148,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -180,6 +188,8 @@ <name>Enrich fqdn2</name> <position x="368.0" y="416.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>0cdf1224-a6d5-3ac4-a4f5-27a7588f1d98</id> <name>Input</name> @@ -199,6 +209,8 @@ <name>Umbrella</name> <position x="379.1725199638772" y="224.3012562119061" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>0306116f-b13d-30fc-94b1-34f3c8ba95da</id> <name>Umbrella fqdn1</name> @@ -209,7 +221,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -338,6 +350,8 @@ <name>Misp lookup</name> <position x="1024.0" y="224.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>cadc3f2e-8e08-195c-898e-1b5fbcdf56e0</id> <name>Misp fqdn2</name> @@ -348,7 +362,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -538,6 +552,8 @@ <name>Enrich fqdn1</name> <position x="376.0" y="200.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>f2cc04f8-56bc-3adb-8d72-6ad7f6e6e48c</id> <name>Input</name> @@ -557,6 +573,8 @@ <name>Umbrella</name> <position x="379.1725199638772" y="224.3012562119061" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>ce84ada1-58b8-3c28-bc5a-64fc4f39e008</id> <name>Umbrella fqdn1</name> @@ -567,7 +585,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -696,6 +714,8 @@ <name>Misp lookup</name> <position x="1056.0" y="216.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>16d53f41-fc96-1292-b434-9a157e27eaf3</id> <name>Misp fqdn1</name> @@ -706,7 +726,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -997,6 +1017,8 @@ <name>Custom enrichment</name> <position x="-536.0" y="944.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>2ba9db29-0172-1000-ffff-ffffc1aa6db2</id> <name>Input</name> @@ -1037,6 +1059,8 @@ <name>IP enrichment</name> <position x="-216.0" y="16.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>be9226d7-777f-3820-bf10-49e97e9b73cc</id> <name>Check IP2 enrichment</name> @@ -1047,7 +1071,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1087,6 +1111,8 @@ <name>Enrich IP1</name> <position x="-48.0" y="312.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>ff302e29-64bb-3e10-b76e-91a13d2470d8</id> <name>Input</name> @@ -1106,6 +1132,8 @@ <name>Misp lookup</name> <position x="1160.0" y="136.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>68dc57d7-2016-3e27-ac1f-092c63909a63</id> <name>Misp ip1</name> @@ -1116,7 +1144,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>16</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1245,6 +1273,8 @@ <name>GeoIP</name> <position x="480.0" y="144.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>fb1a97b2-5e15-3bca-9284-f23a9cc90eed</id> <name>Add GeoIP fields</name> @@ -1255,7 +1285,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1314,7 +1344,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-enrich-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>12</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1385,7 +1415,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1575,6 +1605,8 @@ <name>Tor nodes lookup</name> <position x="480.0" y="384.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>fd52d735-a256-3c52-9b90-fbe71f010fbe</id> <name>Tor src IP</name> @@ -1585,7 +1617,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>12</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1607,7 +1639,7 @@ </property> <property> <name>lookup-service</name> - <value>bf81debc-0171-1000-0000-00002936ae5a</value> + <value>bbd4d3a2-0175-1000-0000-00000b0fb8bd</value> </property> <property> <name>result-record-path</name> @@ -1795,6 +1827,8 @@ <name>Enrich IP2</name> <position x="-56.0" y="584.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>742a8211-a5e6-347d-9e79-25facc6b181e</id> <name>Input</name> @@ -1814,6 +1848,8 @@ <name>GeoIP and IPreg</name> <position x="480.0" y="144.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>9ed935e9-7bd4-150a-913d-e6a4dedb2939</id> <name>Contry code to region</name> @@ -1824,7 +1860,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1879,7 +1915,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-enrich-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>16</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -1950,7 +1986,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -2144,6 +2180,8 @@ <name>Tor nodes lookup</name> <position x="480.0" y="384.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>adae2d07-ad4f-38f2-9a8c-b7638863cac7</id> <name>Tor src IP</name> @@ -2154,7 +2192,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>16</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -2176,7 +2214,7 @@ </property> <property> <name>lookup-service</name> - <value>bf81debc-0171-1000-0000-00002936ae5a</value> + <value>bbd4d3a2-0175-1000-0000-00000b0fb8bd</value> </property> <property> <name>result-record-path</name> @@ -2285,6 +2323,8 @@ <name>Misp lookup</name> <position x="1160.0" y="136.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>2cd2029e-53ae-3575-bf35-785203683c7f</id> <name>Misp ip</name> @@ -2295,7 +2335,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>12</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -2606,6 +2646,8 @@ <name>Domain enrichment</name> <position x="-213.29739929099082" y="283.59665734060223" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>1957f5c3-b6cb-3c80-b527-1858c61ed111</id> <name>Check domain2 enrichment</name> @@ -2616,7 +2658,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -2656,6 +2698,8 @@ <name>Enrich domain1</name> <position x="448.0" y="248.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>bf74c1ba-45ad-3c0d-84c6-ec7ea7bdcdea</id> <name>Input</name> @@ -2675,6 +2719,8 @@ <name>Entropy</name> <position x="1152.0" y="288.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>bf907759-0171-1000-0000-0000022727cd</id> <name>Calculate entropy on domain1</name> @@ -2685,7 +2731,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-scripting-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -2821,6 +2867,8 @@ <name>Misp lookup</name> <position x="456.0" y="552.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>9ed93644-7bd4-150a-0000-00003c8c3388</id> <name>Misp domain1</name> @@ -2831,7 +2879,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -2960,6 +3008,8 @@ <name>Alexa</name> <position x="448.0" y="288.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>ccab62e9-53cd-3d5e-aca3-6fa2e1b95597</id> <name>Alexa domain1</name> @@ -2970,7 +3020,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -3180,6 +3230,8 @@ <name>Enrich domain2</name> <position x="440.0" y="496.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>0097ae66-e4c5-316c-b301-c8fd4481cbf4</id> <name>Input</name> @@ -3199,6 +3251,8 @@ <name>Misp lookup</name> <position x="448.0" y="520.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>2a1b3c82-c06f-1184-b6ec-245bb0032b5a</id> <name>Misp domain1</name> @@ -3209,7 +3263,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -3338,6 +3392,8 @@ <name>Entropy</name> <position x="1152.0" y="288.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>eff73930-f310-1a2e-b248-812b172b7415</id> <name>Calculate entropy on domain2</name> @@ -3348,7 +3404,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-scripting-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -3484,6 +3540,8 @@ <name>Alexa</name> <position x="448.0" y="288.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>4598e8b8-b54a-3855-b6b7-5ac2721745a2</id> <name>Alexa domain1</name> @@ -3494,7 +3552,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>8</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -4028,7 +4086,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-lookup-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -4037,7 +4095,7 @@ </property> <property> <name>CSV Format</name> - <value>Default</value> + <value>default</value> </property> <property> <name>Character Set</name> @@ -4048,13 +4106,36 @@ <value>domain</value> </property> <property> - <name>lookup-value-column</name> - <value>index</value> + <name>ignore-duplicates</name> + <value>true</value> </property> <property> - <name>ignore-duplicates</name> + <name>Value Separator</name> + <value>,</value> + </property> + <property> + <name>Quote Character</name> + <value>"</value> + </property> + <property> + <name>Quote Mode</name> + <value>MINIMAL</value> + </property> + <property> + <name>Comment Marker</name> + </property> + <property> + <name>Escape Character</name> + <value>\</value> + </property> + <property> + <name>Trim Fields</name> <value>true</value> </property> + <property> + <name>lookup-value-column</name> + <value>index</value> + </property> </controllerService> <controllerService> <id>67289e27-a14d-3fa6-bcf9-91f7d2ae8d59</id> @@ -4064,7 +4145,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-lookup-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -4073,7 +4154,7 @@ </property> <property> <name>CSV Format</name> - <value>Excel</value> + <value>default</value> </property> <property> <name>Character Set</name> @@ -4084,13 +4165,36 @@ <value>domain</value> </property> <property> - <name>lookup-value-column</name> - <value>index</value> + <name>ignore-duplicates</name> + <value>true</value> </property> <property> - <name>ignore-duplicates</name> + <name>Value Separator</name> + <value>,</value> + </property> + <property> + <name>Quote Character</name> + <value>"</value> + </property> + <property> + <name>Quote Mode</name> + <value>MINIMAL</value> + </property> + <property> + <name>Comment Marker</name> + </property> + <property> + <name>Escape Character</name> + <value>\</value> + </property> + <property> + <name>Trim Fields</name> <value>true</value> </property> + <property> + <name>lookup-value-column</name> + <value>index</value> + </property> </controllerService> </processGroup> <processGroup> @@ -4098,6 +4202,8 @@ <name>Data input</name> <position x="830.4597621124223" y="407.3463126314215" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <outputPort> <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id> <name>To enrichment</name> @@ -4110,285 +4216,48 @@ <name>To data output</name> <position x="-1120.0" y="592.0" /> <comments /> - <scheduledState>STOPPED</scheduledState> + <scheduledState>RUNNING</scheduledState> </outputPort> <processGroup> <id>27d51d04-0172-1000-0000-00004573c6ec</id> <name>Custom data inputs</name> <position x="-504.0" y="952.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <outputPort> - <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id> - <name>To data output</name> - <position x="-632.0" y="328.0" /> + <id>27d5761b-0172-1000-0000-000059275dad</id> + <name>To enrichment</name> + <position x="-312.0" y="328.0" /> <comments /> <scheduledState>STOPPED</scheduledState> </outputPort> <outputPort> - <id>27d5761b-0172-1000-0000-000059275dad</id> - <name>To enrichment</name> - <position x="-312.0" y="328.0" /> + <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id> + <name>To data output</name> + <position x="-632.0" y="328.0" /> <comments /> <scheduledState>STOPPED</scheduledState> </outputPort> </processGroup> <processGroup> - <id>84607b52-9748-3d38-b519-b0a05cddd097</id> - <name>Nifi logs</name> - <position x="-496.0" y="136.0" /> + <id>0c790562-0175-1000-ffff-ffffeaaeafc3</id> + <name>FileBeat</name> + <position x="-496.0" y="344.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> - <id>c1318ad1-0c35-3896-b32a-1ccd6c09864a</id> - <name>Add source fields and fix timestamp</name> - <position x="-2192.0" y="560.0" /> - <styles /> - <comment /> - <class>org.apache.nifi.processors.standard.UpdateRecord</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> - </bundle> - <maxConcurrentTasks>1</maxConcurrentTasks> - <schedulingPeriod>0 sec</schedulingPeriod> - <penalizationPeriod>30 sec</penalizationPeriod> - <yieldPeriod>1 sec</yieldPeriod> - <bulletinLevel>WARN</bulletinLevel> - <lossTolerant>false</lossTolerant> - <scheduledState>STOPPED</scheduledState> - <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> - <executionNode>ALL</executionNode> - <runDurationNanos>0</runDurationNanos> - <property> - <name>record-reader</name> - <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> - </property> - <property> - <name>record-writer</name> - <value>94600c6c-704e-3ff8-a2a4-f2f25c71dc3b</value> - </property> - <property> - <name>replacement-value-strategy</name> - <value>literal-value</value> - </property> - <property> - <name>/labels/source</name> - <value>${tailfile.original.path}</value> - </property> - <property> - <name>/labels/source_host</name> - <value>${hostname()}</value> - </property> - <property> - <name>/timestamp</name> - <value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value> - </property> - </processor> - <processor> - <id>68663f14-f470-32ee-9cb3-224344d5ad6a</id> - <name>UpdateAttribute</name> - <position x="-2184.0" y="824.0" /> - <styles /> - <comment /> - <class>org.apache.nifi.processors.attributes.UpdateAttribute</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> - </bundle> - <maxConcurrentTasks>1</maxConcurrentTasks> - <schedulingPeriod>0 sec</schedulingPeriod> - <penalizationPeriod>30 sec</penalizationPeriod> - <yieldPeriod>1 sec</yieldPeriod> - <bulletinLevel>WARN</bulletinLevel> - <lossTolerant>false</lossTolerant> - <scheduledState>STOPPED</scheduledState> - <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> - <executionNode>ALL</executionNode> - <runDurationNanos>0</runDurationNanos> - <property> - <name>Delete Attributes Expression</name> - </property> - <property> - <name>Store State</name> - <value>Do not store state</value> - </property> - <property> - <name>Stateful Variables Initial Value</name> - </property> - <property> - <name>canonical-value-lookup-cache-size</name> - <value>100</value> - </property> - <property> - <name>data_index</name> - <value>nifi-logs</value> - </property> - <property> - <name>Authorization</name> - <value>Bearer 874rPdPJ6qOSL6HvWcH8xxChqTJxiFtHp94puxh4MygY</value> - </property> - </processor> - <processor> - <id>716d22cb-8b81-342f-abe4-7cdfe2a980ff</id> - <name>ConvertRecord</name> - <position x="-2200.0" y="264.0" /> - <styles /> - <comment /> - <class>org.apache.nifi.processors.standard.ConvertRecord</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> - </bundle> - <maxConcurrentTasks>1</maxConcurrentTasks> - <schedulingPeriod>0 sec</schedulingPeriod> - <penalizationPeriod>30 sec</penalizationPeriod> - <yieldPeriod>1 sec</yieldPeriod> - <bulletinLevel>WARN</bulletinLevel> - <lossTolerant>false</lossTolerant> - <scheduledState>STOPPED</scheduledState> - <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> - <executionNode>ALL</executionNode> - <runDurationNanos>0</runDurationNanos> - <property> - <name>record-reader</name> - <value>e3e44ca0-6653-328b-9d3f-b8225312914b</value> - </property> - <property> - <name>record-writer</name> - <value>94600c6c-704e-3ff8-a2a4-f2f25c71dc3b</value> - </property> - <property> - <name>include-zero-record-flowfiles</name> - <value>true</value> - </property> - </processor> - <processor> - <id>609a3835-5317-1c94-ad8f-1d9940869db4</id> - <name>TailFile</name> - <position x="-2200.0" y="8.0" /> - <styles /> - <comment /> - <class>org.apache.nifi.processors.standard.TailFile</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> - </bundle> - <maxConcurrentTasks>1</maxConcurrentTasks> - <schedulingPeriod>0 sec</schedulingPeriod> - <penalizationPeriod>30 sec</penalizationPeriod> - <yieldPeriod>1 sec</yieldPeriod> - <bulletinLevel>WARN</bulletinLevel> - <lossTolerant>false</lossTolerant> - <scheduledState>STOPPED</scheduledState> - <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> - <executionNode>ALL</executionNode> - <runDurationNanos>0</runDurationNanos> - <property> - <name>tail-mode</name> - <value>Single file</value> - </property> - <property> - <name>File to Tail</name> - <value>/opt/nifi/nifi-current/logs/nifi-user.log</value> - </property> - <property> - <name>Rolling Filename Pattern</name> - </property> - <property> - <name>tail-base-directory</name> - </property> - <property> - <name>Initial Start Position</name> - <value>Beginning of File</value> - </property> - <property> - <name>File Location</name> - <value>Local</value> - </property> - <property> - <name>tailfile-recursive-lookup</name> - <value>false</value> - </property> - <property> - <name>tailfile-lookup-frequency</name> - <value>10 minutes</value> - </property> - <property> - <name>tailfile-maximum-age</name> - <value>24 hours</value> - </property> - </processor> - <processor> - <id>9ecf3ce4-ae3c-1b06-96f9-5e9c59e51690</id> - <name>TailFile</name> - <position x="-1784.0" y="8.0" /> - <styles /> - <comment /> - <class>org.apache.nifi.processors.standard.TailFile</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> - </bundle> - <maxConcurrentTasks>1</maxConcurrentTasks> - <schedulingPeriod>0 sec</schedulingPeriod> - <penalizationPeriod>30 sec</penalizationPeriod> - <yieldPeriod>1 sec</yieldPeriod> - <bulletinLevel>WARN</bulletinLevel> - <lossTolerant>false</lossTolerant> - <scheduledState>STOPPED</scheduledState> - <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> - <executionNode>ALL</executionNode> - <runDurationNanos>0</runDurationNanos> - <property> - <name>tail-mode</name> - <value>Single file</value> - </property> - <property> - <name>File to Tail</name> - <value>/opt/nifi/nifi-current/logs/nifi-bootstrap.log</value> - </property> - <property> - <name>Rolling Filename Pattern</name> - </property> - <property> - <name>tail-base-directory</name> - </property> - <property> - <name>Initial Start Position</name> - <value>Beginning of File</value> - </property> - <property> - <name>File Location</name> - <value>Local</value> - </property> - <property> - <name>tailfile-recursive-lookup</name> - <value>false</value> - </property> - <property> - <name>tailfile-lookup-frequency</name> - <value>10 minutes</value> - </property> - <property> - <name>tailfile-maximum-age</name> - <value>24 hours</value> - </property> - </processor> - <processor> - <id>1e796098-c064-371a-8147-d60b5d41a316</id> - <name>TailFile</name> - <position x="-2648.0" y="16.0" /> + <id>8962ad5a-0175-1000-ffff-ffffde6db5a6</id> + <name>RouteOnAttribute</name> + <position x="-1080.0" y="280.0" /> <styles /> <comment /> - <class>org.apache.nifi.processors.standard.TailFile</class> + <class>org.apache.nifi.processors.standard.RouteOnAttribute</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -4396,305 +4265,38 @@ <yieldPeriod>1 sec</yieldPeriod> <bulletinLevel>WARN</bulletinLevel> <lossTolerant>false</lossTolerant> - <scheduledState>STOPPED</scheduledState> + <scheduledState>RUNNING</scheduledState> <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> <executionNode>ALL</executionNode> <runDurationNanos>0</runDurationNanos> <property> - <name>tail-mode</name> - <value>Single file</value> - </property> - <property> - <name>File to Tail</name> - <value>/opt/nifi/nifi-current/logs/nifi-app.log</value> - </property> - <property> - <name>Rolling Filename Pattern</name> - </property> - <property> - <name>tail-base-directory</name> - </property> - <property> - <name>Initial Start Position</name> - <value>Beginning of File</value> - </property> - <property> - <name>File Location</name> - <value>Local</value> + <name>Routing Strategy</name> + <value>Route to Property name</value> </property> <property> - <name>tailfile-recursive-lookup</name> - <value>false</value> + <name>suricata</name> + <value>${log_type:equals("suricata")}</value> </property> <property> - <name>tailfile-lookup-frequency</name> - <value>10 minutes</value> + <name>zeek</name> + <value>${log_type:equals("zeek")}</value> </property> <property> - <name>tailfile-maximum-age</name> - <value>24 hours</value> + <name>nifi</name> + <value>${log_type:equals("nifi")}</value> </property> </processor> <outputPort> - <id>1ef39440-1985-3bbb-8e03-859a1c5ee4b1</id> - <name>To storage</name> - <position x="-2120.0" y="1064.0" /> + <id>bcb879d5-0175-1000-0000-000070879ad0</id> + <name>To data output</name> + <position x="-2480.0" y="336.0" /> <comments /> - <scheduledState>STOPPED</scheduledState> + <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>beabd3be-6f95-3369-9aa5-4631e6207ec5</id> - <position x="-1572.2279720213353" y="320.16022816068823" /> - </funnel> - <connection> - <id>0261c39c-d823-3ab3-b680-2d0c2fa152e5</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>c1318ad1-0c35-3896-b32a-1ccd6c09864a</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>68663f14-f470-32ee-9cb3-224344d5ad6a</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>PROCESSOR</destinationType> - <relationship>success</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>214adcbf-0175-1000-ffff-ffffaedeecef</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>9ecf3ce4-ae3c-1b06-96f9-5e9c59e51690</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>PROCESSOR</destinationType> - <relationship>success</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>0a34d026-0761-3a6d-b261-377d20a40b5d</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>c1318ad1-0c35-3896-b32a-1ccd6c09864a</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>beabd3be-6f95-3369-9aa5-4631e6207ec5</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>FUNNEL</destinationType> - <relationship>failure</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>266ad4f3-c827-337a-b3b5-28bccc8917fc</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>beabd3be-6f95-3369-9aa5-4631e6207ec5</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>FUNNEL</destinationType> - <relationship>failure</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>0dc021fb-a375-3e48-a5fd-1b9d5c3ad9bd</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>1e796098-c064-371a-8147-d60b5d41a316</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>PROCESSOR</destinationType> - <relationship>success</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>5bc3c82f-bfb5-3e6c-a7bf-141748391beb</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>c1318ad1-0c35-3896-b32a-1ccd6c09864a</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>PROCESSOR</destinationType> - <relationship>success</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>32df885e-4eb1-38fd-bf6b-219725264cf4</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>68663f14-f470-32ee-9cb3-224344d5ad6a</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>1ef39440-1985-3bbb-8e03-859a1c5ee4b1</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>OUTPUT_PORT</destinationType> - <relationship>success</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <connection> - <id>609a3840-5317-1c94-0000-00002897f5ab</id> - <name /> - <bendPoints /> - <labelIndex>1</labelIndex> - <zIndex>0</zIndex> - <sourceId>609a3835-5317-1c94-ad8f-1d9940869db4</sourceId> - <sourceGroupId>84607b52-9748-3d38-b519-b0a05cddd097</sourceGroupId> - <sourceType>PROCESSOR</sourceType> - <destinationId>716d22cb-8b81-342f-abe4-7cdfe2a980ff</destinationId> - <destinationGroupId>84607b52-9748-3d38-b519-b0a05cddd097</destinationGroupId> - <destinationType>PROCESSOR</destinationType> - <relationship>success</relationship> - <maxWorkQueueSize>10000</maxWorkQueueSize> - <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> - <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> - <partitioningAttribute /> - <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> - </connection> - <controllerService> - <id>e3e44ca0-6653-328b-9d3f-b8225312914b</id> - <name>Nifi logs GrokReader</name> - <comment /> - <class>org.apache.nifi.grok.GrokReader</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-record-serialization-services-nar</artifact> - <version>1.11.4</version> - </bundle> - <enabled>true</enabled> - <property> - <name>schema-access-strategy</name> - <value>string-fields-from-grok-expression</value> - </property> - <property> - <name>schema-registry</name> - </property> - <property> - <name>schema-name</name> - <value>${schema.name}</value> - </property> - <property> - <name>schema-version</name> - </property> - <property> - <name>schema-branch</name> - </property> - <property> - <name>schema-text</name> - <value>${avro.schema}</value> - </property> - <property> - <name>Grok Pattern File</name> - </property> - <property> - <name>Grok Expression</name> - <value>%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:class} %{GREEDYDATA:message}</value> - </property> - <property> - <name>no-match-behavior</name> - <value>append-to-previous-message</value> - </property> - </controllerService> - </processGroup> - <processGroup> - <id>0c790562-0175-1000-ffff-ffffeaaeafc3</id> - <name>FileBeat</name> - <position x="-496.0" y="344.0" /> - <comment /> - <processor> - <id>8962ad5a-0175-1000-ffff-ffffde6db5a6</id> - <name>RouteOnAttribute</name> - <position x="-1080.0" y="280.0" /> - <styles /> - <comment /> - <class>org.apache.nifi.processors.standard.RouteOnAttribute</class> - <bundle> - <group>org.apache.nifi</group> - <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> - </bundle> - <maxConcurrentTasks>1</maxConcurrentTasks> - <schedulingPeriod>0 sec</schedulingPeriod> - <penalizationPeriod>30 sec</penalizationPeriod> - <yieldPeriod>1 sec</yieldPeriod> - <bulletinLevel>WARN</bulletinLevel> - <lossTolerant>false</lossTolerant> - <scheduledState>RUNNING</scheduledState> - <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> - <executionNode>ALL</executionNode> - <runDurationNanos>0</runDurationNanos> - <property> - <name>Routing Strategy</name> - <value>Route to Property name</value> - </property> - <property> - <name>suricata</name> - <value>${log_type:equals("suricata")}</value> - </property> - <property> - <name>zeek</name> - <value>${log_type:equals("zeek")}</value> - </property> - </processor> <outputPort> <id>349b32fe-a821-1197-0000-00003a0b6fe5</id> - <name>Output</name> - <position x="272.0" y="320.0" /> + <name>To enrichment</name> + <position x="352.0" y="296.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> @@ -4703,17 +4305,192 @@ <name>Unknown data</name> <position x="-448.0" y="64.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> + <processor> + <id>8963d0f9-0175-1000-0000-000054fbe086</id> + <name>UpdateAttribute</name> + <position x="392.0" y="248.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.attributes.UpdateAttribute</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-update-attribute-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Delete Attributes Expression</name> + </property> + <property> + <name>Store State</name> + <value>Do not store state</value> + </property> + <property> + <name>Stateful Variables Initial Value</name> + </property> + <property> + <name>canonical-value-lookup-cache-size</name> + <value>100</value> + </property> + <property> + <name>data_index</name> + <value>logs-filebeat-unknown</value> + </property> + </processor> + <inputPort> + <id>89639d3d-0175-1000-ffff-ffffb446c257</id> + <name>Input</name> + <position x="444.0000243687773" y="80.00000220501622" /> + <comments /> + <scheduledState>RUNNING</scheduledState> + </inputPort> + <outputPort> + <id>8963b202-0175-1000-0000-000022d64ba2</id> + <name>Output</name> + <position x="456.0" y="504.0" /> + <comments /> + <scheduledState>RUNNING</scheduledState> + </outputPort> + <connection> + <id>8963e649-0175-1000-ffff-fffff03ab629</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>89639d3d-0175-1000-ffff-ffffb446c257</sourceId> + <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId> + <sourceType>INPUT_PORT</sourceType> + <destinationId>8963d0f9-0175-1000-0000-000054fbe086</destinationId> + <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship /> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>8963f112-0175-1000-0000-00000dfa15b5</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>8963d0f9-0175-1000-0000-000054fbe086</sourceId> + <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>8963b202-0175-1000-0000-000022d64ba2</destinationId> + <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId> + <destinationType>OUTPUT_PORT</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + </processGroup> + <processGroup> + <id>bc6be78f-0175-1000-ffff-ffffbcd0f569</id> + <name>NiFi logs</name> + <position x="-1904.0" y="264.0" /> + <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> + <processor> + <id>bc903708-0175-1000-0000-0000642abebf</id> + <name>Extract message</name> + <position x="352.0" y="280.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.ConvertRecord</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>record-reader</name> + <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> + </property> + <property> + <name>record-writer</name> + <value>bc8e5957-0175-1000-0000-00003346421d</value> + </property> + <property> + <name>include-zero-record-flowfiles</name> + <value>true</value> + </property> + </processor> <processor> - <id>8963d0f9-0175-1000-0000-000054fbe086</id> + <id>bc91c66f-0175-1000-0000-00005c7f88ce</id> + <name>Convert to json</name> + <position x="1064.0" y="272.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.ConvertRecord</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>record-reader</name> + <value>bc97858d-0175-1000-0000-0000130a84f8</value> + </property> + <property> + <name>record-writer</name> + <value>17b30955-5464-3709-8a32-69a459850cfa</value> + </property> + <property> + <name>include-zero-record-flowfiles</name> + <value>true</value> + </property> + </processor> + <processor> + <id>bcabbf11-0175-1000-0000-000037f4e3d3</id> <name>UpdateAttribute</name> - <position x="392.0" y="248.0" /> + <position x="1072.0" y="472.0" /> <styles /> <comment /> <class>org.apache.nifi.processors.attributes.UpdateAttribute</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -4741,34 +4518,129 @@ </property> <property> <name>data_index</name> - <value>logs-filebeat-unknown</value> + <value>logs-nifi</value> + </property> + </processor> + <processor> + <id>bc9ffeb0-0175-1000-0000-00000a88d684</id> + <name>UpdateRecord</name> + <position x="352.0" y="472.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.UpdateRecord</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>record-reader</name> + <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> + </property> + <property> + <name>record-writer</name> + <value>17b30955-5464-3709-8a32-69a459850cfa</value> + </property> + <property> + <name>replacement-value-strategy</name> + <value>literal-value</value> + </property> + <property> + <name>/labels/source_file</name> + <value>${source_file}</value> + </property> + <property> + <name>/labels/source_host</name> + <value>${source_host}</value> + </property> + <property> + <name>/timestamp</name> + <value>${field.value:toDate('yyyy-MM-dd HH:mm:ss,SSS'):format("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")}</value> </property> </processor> <inputPort> - <id>89639d3d-0175-1000-ffff-ffffb446c257</id> + <id>bc6c2159-0175-1000-ffff-ffffb4de4d47</id> <name>Input</name> - <position x="444.0000243687773" y="80.00000220501622" /> + <position x="397.9999517774115" y="110.99999315685733" /> <comments /> <scheduledState>RUNNING</scheduledState> </inputPort> <outputPort> - <id>8963b202-0175-1000-0000-000022d64ba2</id> + <id>bca9636a-0175-1000-0000-000013fa95aa</id> <name>Output</name> - <position x="456.0" y="504.0" /> + <position x="1120.0" y="808.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> + <funnel> + <id>bc925474-0175-1000-0000-00004e78071f</id> + <position x="1882.9999517774115" y="327.9999931568573" /> + </funnel> + <funnel> + <id>bc90d189-0175-1000-0000-0000037bc986</id> + <position x="8.0" y="424.0" /> + </funnel> <connection> - <id>8963e649-0175-1000-ffff-fffff03ab629</id> + <id>bc90c7ac-0175-1000-ffff-fffffa80b534</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>89639d3d-0175-1000-ffff-ffffb446c257</sourceId> - <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId> + <sourceId>bc903708-0175-1000-0000-0000642abebf</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bc91c66f-0175-1000-0000-00005c7f88ce</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bc924694-0175-1000-0000-00005b0604b6</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bc91c66f-0175-1000-0000-00005c7f88ce</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bc9ffeb0-0175-1000-0000-00000a88d684</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bc6e1b20-0175-1000-ffff-ffff9e7dcb75</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bc6c2159-0175-1000-ffff-ffffb4de4d47</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> <sourceType>INPUT_PORT</sourceType> - <destinationId>8963d0f9-0175-1000-0000-000054fbe086</destinationId> - <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId> + <destinationId>bc903708-0175-1000-0000-0000642abebf</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship /> <maxWorkQueueSize>10000</maxWorkQueueSize> @@ -4779,16 +4651,76 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> - <id>8963f112-0175-1000-0000-00000dfa15b5</id> + <id>bc92a603-0175-1000-ffff-ffff9dadafd9</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>8963d0f9-0175-1000-0000-000054fbe086</sourceId> - <sourceGroupId>89636688-0175-1000-ffff-ffffb1b28a38</sourceGroupId> + <sourceId>bc91c66f-0175-1000-0000-00005c7f88ce</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>8963b202-0175-1000-0000-000022d64ba2</destinationId> - <destinationGroupId>89636688-0175-1000-ffff-ffffb1b28a38</destinationGroupId> + <destinationId>bc925474-0175-1000-0000-00004e78071f</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>60 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bcad0899-0175-1000-ffff-ffffa8a1a378</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bc9ffeb0-0175-1000-0000-00000a88d684</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bc90d189-0175-1000-0000-0000037bc986</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>60 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bc90e481-0175-1000-ffff-ffffca3753fb</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bc903708-0175-1000-0000-0000642abebf</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bc90d189-0175-1000-0000-0000037bc986</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>60 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bcad2e36-0175-1000-0000-00002b6e8fe7</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bcabbf11-0175-1000-0000-000037f4e3d3</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bca9636a-0175-1000-0000-000013fa95aa</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> <destinationType>OUTPUT_PORT</destinationType> <relationship>success</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> @@ -4798,12 +4730,34 @@ <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> + <connection> + <id>bca97855-0175-1000-ffff-ffffbd18cb66</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bc9ffeb0-0175-1000-0000-00000a88d684</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bcabbf11-0175-1000-0000-000037f4e3d3</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> </processGroup> <processGroup> <id>895eab20-0175-1000-0000-00007e13267d</id> <name>Common ListenBeats</name> <position x="-1096.0" y="0.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>71be315f-7e16-1cce-89f1-d5bd502f889f</id> <name>Prepend [</name> @@ -4814,7 +4768,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -4840,7 +4794,7 @@ </property> <property> <name>Maximum Buffer Size</name> - <value>1 MB</value> + <value>2 MB</value> </property> <property> <name>Replacement Strategy</name> @@ -4865,7 +4819,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-beats-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -4931,7 +4885,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -4957,7 +4911,7 @@ </property> <property> <name>Maximum Buffer Size</name> - <value>1 MB</value> + <value>2 MB</value> </property> <property> <name>Replacement Strategy</name> @@ -4982,7 +4936,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-beats-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5003,7 +4957,7 @@ </property> <property> <name>Receive Buffer Size</name> - <value>65507 B</value> + <value>1024kb</value> </property> <property> <name>Max Size of Message Queue</name> @@ -5011,7 +4965,7 @@ </property> <property> <name>Max Size of Socket Buffer</name> - <value>2 MB</value> + <value>4 MB</value> </property> <property> <name>Character Set</name> @@ -5019,7 +4973,7 @@ </property> <property> <name>Max Batch Size</name> - <value>10000</value> + <value>500</value> </property> <property> <name>Message Delimiter</name> @@ -5027,7 +4981,7 @@ </property> <property> <name>Max Number of TCP Connections</name> - <value>100</value> + <value>200</value> </property> <property> <name>SSL_CONTEXT_SERVICE</name> @@ -5037,17 +4991,71 @@ <value>NONE</value> </property> </processor> + <processor> + <id>bb719fee-0175-1000-ffff-ffffb73dd31a</id> + <name>Rename @ fields</name> + <position x="-1080.0" y="360.0" /> + <styles /> + <comment /> + <class>org.apache.nifi.processors.standard.JoltTransformJSON</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-standard-nar</artifact> + <version>1.12.1</version> + </bundle> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <executionNode>ALL</executionNode> + <runDurationNanos>0</runDurationNanos> + <property> + <name>jolt-transform</name> + <value>jolt-transform-chain</value> + </property> + <property> + <name>jolt-custom-class</name> + </property> + <property> + <name>jolt-custom-modules</name> + </property> + <property> + <name>jolt-spec</name> + <value>[{ + "operation": "shift", + "spec": { + "*": { + "\\@timestamp":"[&1].timestamp", + "\\@metadata":"[&1].metadata", + "*": "[&1].&" + } + } +}]</value> + </property> + <property> + <name>Transform Cache Size</name> + <value>1</value> + </property> + <property> + <name>pretty_print</name> + <value>false</value> + </property> + </processor> <processor> <id>896047e7-0175-1000-ffff-ffffc69204e4</id> <name>PartitionRecord</name> - <position x="-1080.0" y="384.0" /> + <position x="-424.0" y="368.0" /> <styles /> <comment /> <class>org.apache.nifi.processors.standard.PartitionRecord</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5071,12 +5079,20 @@ <name>log_type</name> <value>/fields/log_type</value> </property> + <property> + <name>source_host</name> + <value>/host/name</value> + </property> + <property> + <name>source_file</name> + <value>/log/file/path</value> + </property> <autoTerminatedRelationship>original</autoTerminatedRelationship> </processor> <outputPort> <id>89620b1a-0175-1000-0000-000078566f34</id> <name>Output</name> - <position x="-1008.0" y="672.0" /> + <position x="-360.0" y="704.0" /> <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> @@ -5089,21 +5105,41 @@ <position x="278.84829417593915" y="332.4492766741185" /> </funnel> <connection> - <id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id> + <id>895fbf8f-0175-1000-ffff-ffffa5d2d01e</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</sourceId> + <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>895faa7a-0175-1000-0000-000014ef9dd3</destinationId> + <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>60 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bb8aafca-0175-1000-0000-000038f8e9fc</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</sourceId> + <sourceId>896047e7-0175-1000-ffff-ffffc69204e4</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>895faa7a-0175-1000-0000-000014ef9dd3</destinationId> + <destinationId>89620b1a-0175-1000-0000-000078566f34</destinationId> <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> - <destinationType>FUNNEL</destinationType> - <relationship>failure</relationship> + <destinationType>OUTPUT_PORT</destinationType> + <relationship>success</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>60 sec</flowFileExpiration> + <flowFileExpiration>0 sec</flowFileExpiration> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> @@ -5129,17 +5165,17 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> - <id>89621ea4-0175-1000-0000-0000096abd35</id> + <id>895f18a7-0175-1000-ffff-ffffbc2237fd</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>896047e7-0175-1000-ffff-ffffc69204e4</sourceId> + <sourceId>71be315f-7e16-1cce-89f1-d5bd502f889f</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>89620b1a-0175-1000-0000-000078566f34</destinationId> + <destinationId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</destinationId> <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> - <destinationType>OUTPUT_PORT</destinationType> + <destinationType>PROCESSOR</destinationType> <relationship>success</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> @@ -5149,15 +5185,15 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> - <id>895f18a7-0175-1000-ffff-ffffbc2237fd</id> + <id>895ee440-0175-1000-ffff-ffffd3ff3143</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>71be315f-7e16-1cce-89f1-d5bd502f889f</sourceId> + <sourceId>d64f3acd-54a6-1b39-b1af-cc0a26156d5b</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</destinationId> + <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId> <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship>success</relationship> @@ -5169,12 +5205,32 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> - <id>896240f8-0175-1000-ffff-ffffb9871577</id> + <id>ac7a988a-0175-1000-ffff-ffff86c66751</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>896047e7-0175-1000-ffff-ffffc69204e4</sourceId> + <sourceId>d3e43667-10ef-1528-b935-47c2f077f2c9</sourceId> + <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId> + <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> + <connection> + <id>bb8a3db4-0175-1000-ffff-ffffec520565</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bb719fee-0175-1000-ffff-ffffb73dd31a</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> <destinationId>895f7db3-0175-1000-ffff-ffff8229d688</destinationId> @@ -5189,15 +5245,15 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> - <id>895ee440-0175-1000-ffff-ffffd3ff3143</id> + <id>bb8a6c6c-0175-1000-0000-00000abdc8f9</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>d64f3acd-54a6-1b39-b1af-cc0a26156d5b</sourceId> + <sourceId>bb719fee-0175-1000-ffff-ffffb73dd31a</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId> + <destinationId>896047e7-0175-1000-ffff-ffffc69204e4</destinationId> <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship>success</relationship> @@ -5209,21 +5265,21 @@ <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> - <id>ac7a988a-0175-1000-ffff-ffff86c66751</id> + <id>bb8a959d-0175-1000-0000-00005213775d</id> <name /> <bendPoints /> <labelIndex>1</labelIndex> <zIndex>0</zIndex> - <sourceId>d3e43667-10ef-1528-b935-47c2f077f2c9</sourceId> + <sourceId>896047e7-0175-1000-ffff-ffffc69204e4</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>71be315f-7e16-1cce-89f1-d5bd502f889f</destinationId> + <destinationId>895faa7a-0175-1000-0000-000014ef9dd3</destinationId> <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> - <destinationType>PROCESSOR</destinationType> - <relationship>success</relationship> + <destinationType>FUNNEL</destinationType> + <relationship>failure</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> - <flowFileExpiration>0 sec</flowFileExpiration> + <flowFileExpiration>60 sec</flowFileExpiration> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> @@ -5237,7 +5293,7 @@ <sourceId>6b9a3cb4-e697-1540-a5fb-ea71cfce8f41</sourceId> <sourceGroupId>895eab20-0175-1000-0000-00007e13267d</sourceGroupId> <sourceType>PROCESSOR</sourceType> - <destinationId>896047e7-0175-1000-ffff-ffffc69204e4</destinationId> + <destinationId>bb719fee-0175-1000-ffff-ffffb73dd31a</destinationId> <destinationGroupId>895eab20-0175-1000-0000-00007e13267d</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship>success</relationship> @@ -5254,6 +5310,8 @@ <name>Suricata</name> <position x="-448.0" y="264.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>8d1bef35-0175-1000-0000-0000746fa33d</id> <name>RouteOnAttribute</name> @@ -5264,7 +5322,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5299,7 +5357,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5368,7 +5426,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5404,7 +5462,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5451,7 +5509,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5516,19 +5574,21 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>d8f19295-5666-31a8-b701-52214c4db51d</id> - <position x="-1500.995244929405" y="257.20806784146276" /> - </funnel> <funnel> <id>9e3adb6e-2266-390c-995d-76bc3aa5c3d8</id> <position x="283.72871497338747" y="273.4623850295515" /> </funnel> + <funnel> + <id>d8f19295-5666-31a8-b701-52214c4db51d</id> + <position x="-1500.995244929405" y="257.20806784146276" /> + </funnel> <processGroup> <id>8d1afcd0-0175-1000-ffff-ffffb3690a74</id> <name>TLS events</name> <position x="-384.0" y="872.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>9279850b-0175-1000-0000-00001e74d182</id> <name>Copy SNI</name> @@ -5539,7 +5599,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5586,7 +5646,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5633,7 +5693,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5812,6 +5872,8 @@ <name>DNS events</name> <position x="-1000.0" y="872.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>8d37fe91-0175-1000-ffff-ffffb5c4de34</id> <name>Add rrname_domain++</name> @@ -5822,7 +5884,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5869,7 +5931,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5900,7 +5962,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5936,7 +5998,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -5981,7 +6043,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -6032,14 +6094,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>8d3298f0-0175-1000-ffff-ffffc9f211a7</id> - <position x="56.0" y="280.0" /> - </funnel> <funnel> <id>8d399854-0175-1000-ffff-ffff8272837e</id> <position x="1736.0" y="528.0" /> </funnel> + <funnel> + <id>8d3298f0-0175-1000-ffff-ffffc9f211a7</id> + <position x="56.0" y="280.0" /> + </funnel> <connection> <id>8d3979b7-0175-1000-ffff-ffffe2efe898</id> <name /> @@ -6511,6 +6573,8 @@ <name>Zeek</name> <position x="-448.0" y="464.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>fec43039-de5d-1e3b-850a-5e25d7b93c76</id> <name>UpdateAttribute</name> @@ -6521,7 +6585,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -6578,7 +6642,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -6629,7 +6693,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-jolt-record-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -6720,14 +6784,14 @@ <comments /> <scheduledState>RUNNING</scheduledState> </outputPort> - <funnel> - <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id> - <position x="248.5321508445502" y="703.4412774751572" /> - </funnel> <funnel> <id>06521038-335b-3139-839d-ab43a013ce03</id> <position x="-1557.869726298236" y="758.8984861527665" /> </funnel> + <funnel> + <id>c8c0a13d-0170-1000-ffff-ffff874141fa</id> + <position x="248.5321508445502" y="703.4412774751572" /> + </funnel> <connection> <id>3c739604-b69c-3e86-ba4c-a4739078837c</id> <name /> @@ -6889,6 +6953,26 @@ <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> + <connection> + <id>bc6e50cc-0175-1000-ffff-ffffbd982e0c</id> + <name /> + <bendPoints /> + <labelIndex>0</labelIndex> + <zIndex>0</zIndex> + <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId> + <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>bc6c2159-0175-1000-ffff-ffffb4de4d47</destinationId> + <destinationGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</destinationGroupId> + <destinationType>INPUT_PORT</destinationType> + <relationship>nifi</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> <connection> <id>89630460-0175-1000-0000-00006b5f18c8</id> <name /> @@ -6929,6 +7013,26 @@ <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> + <connection> + <id>bcadaf87-0175-1000-0000-000048464ec3</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bca9636a-0175-1000-0000-000013fa95aa</sourceId> + <sourceGroupId>bc6be78f-0175-1000-ffff-ffffbcd0f569</sourceGroupId> + <sourceType>OUTPUT_PORT</sourceType> + <destinationId>bcb879d5-0175-1000-0000-000070879ad0</destinationId> + <destinationGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</destinationGroupId> + <destinationType>OUTPUT_PORT</destinationType> + <relationship /> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> <connection> <id>8d0ea3d4-0175-1000-0000-0000471b8522</id> <name /> @@ -6995,6 +7099,8 @@ <name>Test data</name> <position x="-496.0" y="552.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <outputPort> <id>d30dc946-251a-307c-8e88-f2262b0bb194</id> <name>To enrichment</name> @@ -7007,6 +7113,8 @@ <name>Suricata</name> <position x="462.0553417896858" y="119.99261716112323" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>bb7dc9ff-2d25-3134-9617-cca3cabe9179</id> <name>Alerts</name> @@ -7017,7 +7125,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>10 sec</schedulingPeriod> @@ -7056,6 +7164,9 @@ <name>character-set</name> <value>UTF-8</value> </property> + <property> + <name>mime-type</name> + </property> <property> <name>mime.type</name> <value>application/json</value> @@ -7099,7 +7210,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>10 sec</schedulingPeriod> @@ -7135,6 +7246,9 @@ <name>character-set</name> <value>UTF-8</value> </property> + <property> + <name>mime-type</name> + </property> <property> <name>mime.type</name> <value>application/json</value> @@ -7277,6 +7391,26 @@ <partitioningAttribute /> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> + <connection> + <id>bcb8ef9d-0175-1000-0000-000017e52ef1</id> + <name /> + <bendPoints /> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>bcb879d5-0175-1000-0000-000070879ad0</sourceId> + <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId> + <sourceType>OUTPUT_PORT</sourceType> + <destinationId>21a9e277-2d80-359a-9c57-cb76d8962e6d</destinationId> + <destinationGroupId>870d6d68-7a0a-3505-8c42-0d6064fe43f6</destinationGroupId> + <destinationType>OUTPUT_PORT</destinationType> + <relationship /> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> + <partitioningAttribute /> + <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> + </connection> <connection> <id>27d64272-0172-1000-0000-000079e1c9c6</id> <name /> @@ -7329,7 +7463,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-record-serialization-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -7339,6 +7473,10 @@ <property> <name>schema-cache</name> </property> + <property> + <name>schema-protocol-version</name> + <value>1</value> + </property> <property> <name>schema-access-strategy</name> <value>inherit-record-schema</value> @@ -7398,7 +7536,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-record-serialization-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>false</enabled> <property> @@ -7412,6 +7550,8 @@ <name>Data output</name> <position x="829.4446253936723" y="1015.2711478364996" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>e333b82d-7408-3747-8dd2-46473704e51b</id> <name>Data input</name> @@ -7424,6 +7564,8 @@ <name>Elastic odfe</name> <position x="-759.1319580078125" y="739.6137390136719" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>8b48f28f-2379-3f0f-81fe-4e1b93e72666</id> <name>PutElasticsearchHttpRecord</name> @@ -7434,7 +7576,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-elasticsearch-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -7464,7 +7606,7 @@ </property> <property> <name>Password</name> - <value>enc{7b1cf5755da3904a9ab1c4b554431a538ca1409f1f9b94c41e7bab88e58aeb5528fd5bb034229065911e35d565c83d5b}</value> + <value>enc{a2a2e69fa7a4563faa60a9e069ae100df4117e257dfa1cdbe21ed388bc4480066d147f741a89f88b121b598a508212f6}</value> </property> <property> <name>elasticsearch-http-connect-timeout</name> @@ -7610,6 +7752,8 @@ <name>Custom output</name> <position x="-160.0" y="736.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <inputPort> <id>2bb31aa5-0172-1000-0000-00000869fb70</id> <name>Input</name> @@ -7727,9 +7871,9 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-distributed-cache-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> - <enabled>false</enabled> + <enabled>true</enabled> <property> <name>Server Hostname</name> <value>localhost</value> @@ -7752,11 +7896,15 @@ <name>Enrichment data</name> <position x="1720.0" y="248.0" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processGroup> <id>a97a2cb2-e5b2-3c82-a365-ebe5139e2be6</id> <name>Top domains</name> <position x="970.3727876614566" y="673.4981494769316" /> <comment>Downloads CSV files containing top domains from Alexa and Umbrella</comment> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>857cd537-4aeb-31fb-9740-0513e6cc46fe</id> <name>Unzip CSV files</name> @@ -7767,7 +7915,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -7799,7 +7947,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -7850,7 +7998,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>1 day</schedulingPeriod> @@ -7925,7 +8073,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>1 day</schedulingPeriod> @@ -8000,7 +8148,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8041,7 +8189,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8258,6 +8406,8 @@ <name>Tor Nodes</name> <position x="968.7335178760902" y="456.9915202898361" /> <comment>Downloads a CSV file of IP addresses used as Tor nodes</comment> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>ad366a87-89d6-38ff-affe-a1f3575faa8a</id> <name>Save to disk</name> @@ -8268,7 +8418,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8319,7 +8469,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>1 day</schedulingPeriod> @@ -8391,7 +8541,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8545,6 +8695,8 @@ <name>Misp</name> <position x="548.9658647769079" y="453.4916238226681" /> <comment>Polls Misp database once every minute and places new IOCs in a NiFi memcache.</comment> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>283bea4b-2774-3f2d-aabe-cf96989e9997</id> <name>Set timestamp as FlowFile content</name> @@ -8555,7 +8707,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8606,7 +8758,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8649,7 +8801,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8681,6 +8833,14 @@ <name>Read Timeout</name> <value>15 secs</value> </property> + <property> + <name>idle-timeout</name> + <value>5 mins</value> + </property> + <property> + <name>max-idle-connections</name> + <value>5</value> + </property> <property> <name>Include Date Header</name> <value>True</value> @@ -8692,6 +8852,9 @@ <property> <name>Attributes to Send</name> </property> + <property> + <name>Useragent</name> + </property> <property> <name>Basic Authentication Username</name> </property> @@ -8764,6 +8927,13 @@ <name>ignore-response-content</name> <value>false</value> </property> + <property> + <name>form-body-form-name</name> + </property> + <property> + <name>set-form-filename</name> + <value>true</value> + </property> <property> <name>Authorization</name> <value>${misp_token}</value> @@ -8781,7 +8951,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8821,7 +8991,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8861,7 +9031,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8906,7 +9076,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>1 minute</schedulingPeriod> @@ -8941,6 +9111,9 @@ <name>character-set</name> <value>UTF-8</value> </property> + <property> + <name>mime-type</name> + </property> <property> <name>lookup_id</name> <value>ip</value> @@ -8956,7 +9129,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -8999,7 +9172,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-update-attribute-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9044,7 +9217,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9076,7 +9249,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9523,6 +9696,8 @@ <name>GeoIP</name> <position x="556.427978515625" y="673.0274658203125" /> <comment /> + <flowfileConcurrency>UNBOUNDED</flowfileConcurrency> + <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <processor> <id>6292665b-f188-3551-b366-95476b5ac36f</id> <name>Save to disk</name> @@ -9533,7 +9708,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9584,7 +9759,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9616,6 +9791,14 @@ <name>Read Timeout</name> <value>15 secs</value> </property> + <property> + <name>idle-timeout</name> + <value>5 mins</value> + </property> + <property> + <name>max-idle-connections</name> + <value>5</value> + </property> <property> <name>Include Date Header</name> <value>True</value> @@ -9627,6 +9810,9 @@ <property> <name>Attributes to Send</name> </property> + <property> + <name>Useragent</name> + </property> <property> <name>Basic Authentication Username</name> </property> @@ -9699,6 +9885,13 @@ <name>ignore-response-content</name> <value>false</value> </property> + <property> + <name>form-body-form-name</name> + </property> + <property> + <name>set-form-filename</name> + <value>true</value> + </property> <autoTerminatedRelationship>Original</autoTerminatedRelationship> </processor> <processor> @@ -9711,7 +9904,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9750,7 +9943,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9782,7 +9975,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> @@ -9814,7 +10007,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>1 week</schedulingPeriod> @@ -9849,6 +10042,9 @@ <name>character-set</name> <value>UTF-8</value> </property> + <property> + <name>mime-type</name> + </property> </processor> <funnel> <id>c2cac6f3-c926-3038-b685-68f71f76fda3</id> @@ -10060,6 +10256,50 @@ </connection> </processGroup> </processGroup> + <controllerService> + <id>bc97858d-0175-1000-0000-0000130a84f8</id> + <name>Nifi logs GrokReader</name> + <comment /> + <class>org.apache.nifi.grok.GrokReader</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-record-serialization-services-nar</artifact> + <version>1.12.1</version> + </bundle> + <enabled>true</enabled> + <property> + <name>schema-access-strategy</name> + <value>string-fields-from-grok-expression</value> + </property> + <property> + <name>schema-registry</name> + </property> + <property> + <name>schema-name</name> + <value>${schema.name}</value> + </property> + <property> + <name>schema-version</name> + </property> + <property> + <name>schema-branch</name> + </property> + <property> + <name>schema-text</name> + <value>${avro.schema}</value> + </property> + <property> + <name>Grok Pattern File</name> + </property> + <property> + <name>Grok Expression</name> + <value>%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:class} %{GREEDYDATA:message}</value> + </property> + <property> + <name>no-match-behavior</name> + <value>append-to-previous-message</value> + </property> + </controllerService> <controllerService> <id>349b34c7-a821-1197-ffff-ffff85d82877</id> <name>Contry code to region</name> @@ -10068,7 +10308,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-lookup-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10077,7 +10317,7 @@ </property> <property> <name>CSV Format</name> - <value>Default</value> + <value>default</value> </property> <property> <name>Character Set</name> @@ -10087,24 +10327,67 @@ <name>lookup-key-column</name> <value>alpha-2</value> </property> + <property> + <name>ignore-duplicates</name> + <value>true</value> + </property> + <property> + <name>Value Separator</name> + <value>,</value> + </property> + <property> + <name>Quote Character</name> + <value>"</value> + </property> + <property> + <name>Quote Mode</name> + <value>MINIMAL</value> + </property> + <property> + <name>Comment Marker</name> + </property> + <property> + <name>Escape Character</name> + <value>\</value> + </property> + <property> + <name>Trim Fields</name> + <value>true</value> + </property> <property> <name>lookup-value-column</name> <value>region</value> </property> + </controllerService> + <controllerService> + <id>bc8e5957-0175-1000-0000-00003346421d</id> + <name>Extract message field</name> + <comment /> + <class>org.apache.nifi.text.FreeFormTextRecordSetWriter</class> + <bundle> + <group>org.apache.nifi</group> + <artifact>nifi-record-serialization-services-nar</artifact> + <version>1.12.1</version> + </bundle> + <enabled>true</enabled> + <property> + <name>Text</name> + <value>${message}</value> + </property> <property> - <name>ignore-duplicates</name> - <value>true</value> + <name>Character Set</name> + <value>UTF-8</value> </property> </controllerService> <controllerService> - <id>bf81debc-0171-1000-0000-00002936ae5a</id> + <id>bbd4d3a2-0175-1000-0000-00000b0fb8bd</id> <name>Tor node CSV</name> <comment /> - <class>org.apache.nifi.lookup.CSVRecordLookupService</class> + <class>org.apache.nifi.lookup.SimpleCsvFileLookupService</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-lookup-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10112,8 +10395,8 @@ <value>/opt/nifi/nifi-current/conf/enrich/tornodes.csv</value> </property> <property> - <name>csv-format</name> - <value>Default</value> + <name>CSV Format</name> + <value>default</value> </property> <property> <name>Character Set</name> @@ -10127,6 +10410,33 @@ <name>ignore-duplicates</name> <value>true</value> </property> + <property> + <name>Value Separator</name> + <value>,</value> + </property> + <property> + <name>Quote Character</name> + <value>"</value> + </property> + <property> + <name>Quote Mode</name> + <value>MINIMAL</value> + </property> + <property> + <name>Comment Marker</name> + </property> + <property> + <name>Escape Character</name> + <value>\</value> + </property> + <property> + <name>Trim Fields</name> + <value>true</value> + </property> + <property> + <name>lookup-value-column</name> + <value>ip_addr</value> + </property> </controllerService> <controllerService> <id>fa06ec39-7782-3ae3-8dfe-71d28c5240c3</id> @@ -10136,7 +10446,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-lookup-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10156,7 +10466,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-distributed-cache-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10183,7 +10493,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-record-serialization-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10193,6 +10503,10 @@ <property> <name>schema-cache</name> </property> + <property> + <name>schema-protocol-version</name> + <value>1</value> + </property> <property> <name>schema-access-strategy</name> <value>inherit-record-schema</value> @@ -10252,7 +10566,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-distributed-cache-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10283,7 +10597,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-ssl-context-service-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property> @@ -10304,7 +10618,7 @@ </property> <property> <name>Truststore Password</name> - <value>enc{5d07dd81d60b85c07f07a91beba50b82f18a29f5846638ded1a5ce72b324fdd7}</value> + <value>enc{03ca02479138f0ae3484435afa13a8d10bdc66b2b9391e10764a3f022ad1f693}</value> </property> <property> <name>Truststore Type</name> @@ -10323,7 +10637,7 @@ <bundle> <group>org.apache.nifi</group> <artifact>nifi-record-serialization-services-nar</artifact> - <version>1.11.4</version> + <version>1.12.1</version> </bundle> <enabled>true</enabled> <property>