Skip to content
Snippets Groups Projects
Commit 95a2aef4 authored by Václav Bartoš's avatar Václav Bartoš
Browse files

NiFi: Changed the processor writing data to Elastic 

- PutElasticsearchHttpRecord (deprecated) -> PutElasticsearchRecord
- Also, Elastic URL set to soctools-opensearches-[12], so data goes directly, not through haproxy
- Reverted change in access.ips, as it's not needed anymore
parent eb054fe8
No related branches found
No related tags found
1 merge request!5Migrate from ODFE to OpenSearch
Hide whitespace changes
Inline Side-by-side
access.ips
+ 1
− 1
View file @ 95a2aef4
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
### OPENSEARCHES - Start ### ### OPENSEARCHES - Start ###
172.22.0.0/16 127.0.0.1
### OPENSEARCHES - End ### ### OPENSEARCHES - End ###
......
roles/nifi/templates/flow.xml.j2
+ 83
− 86
View file @ 95a2aef4
...@@ -9437,21 +9437,21 @@ ...@@ -9437,21 +9437,21 @@
</inputPort> </inputPort>
<processGroup> <processGroup>
<id>7ebf304b-4978-3adc-ac31-470fb76e5029</id> <id>7ebf304b-4978-3adc-ac31-470fb76e5029</id>
<name>Elastic odfe</name> <name>Elastic OpenSearch</name>
<position x="-759.1319580078125" y="739.6137390136719" /> <position x="-760.0" y="744.0" />
<comment /> <comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency> <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy> <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor> <processor>
<id>8b48f28f-2379-3f0f-81fe-4e1b93e72666</id> <id>9c0d14ff-0184-1000-ffff-ffff9e170ecc</id>
<name>PutElasticsearchHttpRecord</name> <name>PutElasticsearchRecord</name>
<position x="-856.2311706542969" y="629.8186340332031" /> <position x="-856.0" y="632.0" />
<styles /> <styles />
<comment /> <comment />
<class>org.apache.nifi.processors.elasticsearch.PutElasticsearchHttpRecord</class> <class>org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord</class>
<bundle> <bundle>
<group>org.apache.nifi</group> <group>org.apache.nifi</group>
<artifact>nifi-elasticsearch-nar</artifact> <artifact>nifi-elasticsearch-restapi-nar</artifact>
<version>1.12.1</version> <version>1.12.1</version>
</bundle> </bundle>
<maxConcurrentTasks>1</maxConcurrentTasks> <maxConcurrentTasks>1</maxConcurrentTasks>
...@@ -9465,89 +9465,43 @@ ...@@ -9465,89 +9465,43 @@
<executionNode>ALL</executionNode> <executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos> <runDurationNanos>0</runDurationNanos>
<property> <property>
<name>elasticsearch-http-url</name> <name>el-rest-fetch-index</name>
<value>${elastic_url}</value> <value>${data_index}-${now():format("yyyy-MM-dd")}</value>
</property>
<property>
<name>SSL Context Service</name>
<value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>Username</name>
<value>${elastic_username}</value>
</property>
<property>
<name>Password</name>
<value>enc{907d07dbc9a93739b87296d143791fa32517e337630f9e98716909aae03b3cbd1e1c116bc9bbe793879e5d5d8d5b2724}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
<value>5 secs</value>
</property>
<property>
<name>elasticsearch-http-response-timeout</name>
<value>15 secs</value>
</property>
<property>
<name>proxy-configuration-service</name>
</property>
<property>
<name>elasticsearch-http-proxy-host</name>
</property>
<property>
<name>elasticsearch-http-proxy-port</name>
</property> </property>
<property> <property>
<name>proxy-username</name> <name>el-rest-type</name>
</property> </property>
<property> <property>
<name>proxy-password</name> <name>el-rest-client-service</name>
<value>9c106232-0184-1000-ffff-fffff9d8684d</value>
</property> </property>
<property> <property>
<name>put-es-record-record-reader</name> <name>put-es-record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value> <value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property> </property>
<property> <property>
<name>put-es-record-record-writer</name> <name>put-es-record-batch-size</name>
</property> <value>100</value>
<property>
<name>put-es-record-log-all-errors</name>
<value>false</value>
</property> </property>
<property> <property>
<name>put-es-record-id-path</name> <name>put-es-record-id-path</name>
</property> </property>
<property> <property>
<name>put-es-record-index</name> <name>put-es-record-index-record-path</name>
<value>${data_index}-${now():format("yyyy-MM-dd")}</value>
</property>
<property>
<name>put-es-record-type</name>
<value>_doc</value>
</property> </property>
<property> <property>
<name>put-es-record-index-op</name> <name>put-es-record-type-record-path</name>
<value>index</value>
</property> </property>
<property> <property>
<name>suppress-nulls</name> <name>put-es-record-log-error-responses</name>
<value>always-suppress</value> <value>true</value>
</property> </property>
<property> <property>
<name>Date Format</name> <name>put-es-record-error-writer</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
</property> </property>
<autoTerminatedRelationship>success</autoTerminatedRelationship> <autoTerminatedRelationship>success</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship> <autoTerminatedRelationship>failure</autoTerminatedRelationship>
<autoTerminatedRelationship>errors</autoTerminatedRelationship>
</processor> </processor>
<inputPort> <inputPort>
<id>e7d34e01-babe-3022-ad9b-a7620e7c0f38</id> <id>e7d34e01-babe-3022-ad9b-a7620e7c0f38</id>
...@@ -9557,21 +9511,18 @@ ...@@ -9557,21 +9511,18 @@
<scheduledState>RUNNING</scheduledState> <scheduledState>RUNNING</scheduledState>
</inputPort> </inputPort>
<connection> <connection>
<id>3280c550-2117-37a6-8b5e-3bc1953fa17e</id> <id>9c0d2a48-0184-1000-ffff-ffffc4b80f44</id>
<name /> <name />
<bendPoints> <bendPoints />
<bendPoint x="-393.2311706542969" y="669.8186340332031" />
<bendPoint x="-393.2311706542969" y="719.8186340332031" />
</bendPoints>
<labelIndex>1</labelIndex> <labelIndex>1</labelIndex>
<zIndex>0</zIndex> <zIndex>0</zIndex>
<sourceId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</sourceId> <sourceId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</sourceId>
<sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId> <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
<sourceType>PROCESSOR</sourceType> <sourceType>INPUT_PORT</sourceType>
<destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId> <destinationId>9c0d14ff-0184-1000-ffff-ffff9e170ecc</destinationId>
<destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId> <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
<destinationType>PROCESSOR</destinationType> <destinationType>PROCESSOR</destinationType>
<relationship>retry</relationship> <relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration> <flowFileExpiration>0 sec</flowFileExpiration>
...@@ -9580,25 +9531,71 @@ ...@@ -9580,25 +9531,71 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<connection> <connection>
<id>5de8f98f-ce46-3565-b0ce-7f8ecf518c53</id> <id>9c0d9724-0184-1000-ffff-ffffe970e040</id>
<name /> <name />
<bendPoints /> <bendPoints>
<bendPoint x="-391.0" y="677.0" />
<bendPoint x="-368.0" y="720.0" />
</bendPoints>
<labelIndex>1</labelIndex> <labelIndex>1</labelIndex>
<zIndex>0</zIndex> <zIndex>0</zIndex>
<sourceId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</sourceId> <sourceId>9c0d14ff-0184-1000-ffff-ffff9e170ecc</sourceId>
<sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId> <sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
<sourceType>INPUT_PORT</sourceType> <sourceType>PROCESSOR</sourceType>
<destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId> <destinationId>9c0d14ff-0184-1000-ffff-ffff9e170ecc</destinationId>
<destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId> <destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
<destinationType>PROCESSOR</destinationType> <destinationType>PROCESSOR</destinationType>
<relationship /> <relationship>retry</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration> <flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute /> <partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection> </connection>
<controllerService>
<id>9c106232-0184-1000-ffff-fffff9d8684d</id>
<name>ElasticSearchClientService</name>
<comment />
<class>org.apache.nifi.elasticsearch.ElasticSearchClientServiceImpl</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-elasticsearch-client-service-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>el-cs-http-hosts</name>
<value>${elastic_urls}</value>
</property>
<property>
<name>el-cs-username</name>
<value>admin</value>
</property>
<property>
<name>el-cs-password</name>
<value>enc{18f431a8121ccd08597b03628773453455e445222fe72b279f9f67a3f2775b5275c6752eb316a1521d7748b314b06932}</value>
</property>
<property>
<name>el-cs-ssl-context-service</name>
</property>
<property>
<name>el-cs-connect-timeout</name>
<value>5000</value>
</property>
<property>
<name>el-cs-socket-timeout</name>
<value>60000</value>
</property>
<property>
<name>el-cs-retry-timeout</name>
<value>60000</value>
</property>
<property>
<name>el-cs-charset</name>
<value>UTF-8</value>
</property>
</controllerService>
</processGroup> </processGroup>
<processGroup> <processGroup>
<id>2bb2f914-0172-1000-0000-0000240c76e4</id> <id>2bb2f914-0172-1000-0000-0000240c76e4</id>
...@@ -11976,7 +11973,7 @@ ...@@ -11976,7 +11973,7 @@
</property> </property>
<property> <property>
<name>Truststore Password</name> <name>Truststore Password</name>
<value>enc{4e264a7abbb4164f1c4fac0757c23fe630b181bc223b6c985c7de2c09ef870dc}</value> <value>enc{b061a02ce6812799b478388abce887518f82325e7db568bd814dc716b1ffb35e}</value>
</property> </property>
<property> <property>
<name>Truststore Type</name> <name>Truststore Type</name>
...@@ -12616,9 +12613,9 @@ ...@@ -12616,9 +12613,9 @@
<variable name="misp_first_interval" value="60d" /> <variable name="misp_first_interval" value="60d" />
<variable name="elastic_username" value="{{ elastic_username }}" /> <variable name="elastic_username" value="{{ elastic_username }}" />
<variable name="misp_url" value="{{ misp_url }}" /> <variable name="misp_url" value="{{ misp_url }}" />
<variable name="elastic_url" value="https://{{ soctoolsproxy }}:9200" /> <variable name="elastic_urls" value="https://soctools-opensearch-1:9200/,https://soctools-opensearch-2:9200/" />
<variable name="elastic_password" value="{{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}" /> <variable name="elastic_password" value="{{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}" />
</rootGroup> </rootGroup>
<controllerServices /> <controllerServices />
<reportingTasks /> <reportingTasks />
</flowController> </flowController>
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment