Skip to content
Snippets Groups Projects
Commit 95a2aef4 authored by Václav Bartoš's avatar Václav Bartoš
Browse files

NiFi: Changed the processor writing data to Elastic 

- PutElasticsearchHttpRecord (deprecated) -> PutElasticsearchRecord
- Also, Elastic URL set to soctools-opensearches-[12], so data goes directly, not through haproxy
- Reverted change in access.ips, as it's not needed anymore
parent eb054fe8
No related branches found
No related tags found
1 merge request!5Migrate from ODFE to OpenSearch
Hide whitespace changes
Inline Side-by-side
access.ips
+ 1
− 1
View file @ 95a2aef4
......@@ -20,7 +20,7 @@
### OPENSEARCHES - Start ###
172.22.0.0/16
127.0.0.1
### OPENSEARCHES - End ###
......
roles/nifi/templates/flow.xml.j2
+ 83
− 86
View file @ 95a2aef4
......@@ -9437,21 +9437,21 @@
</inputPort>
<processGroup>
<id>7ebf304b-4978-3adc-ac31-470fb76e5029</id>
<name>Elastic odfe</name>
<position x="-759.1319580078125" y="739.6137390136719" />
<name>Elastic OpenSearch</name>
<position x="-760.0" y="744.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<processor>
<id>8b48f28f-2379-3f0f-81fe-4e1b93e72666</id>
<name>PutElasticsearchHttpRecord</name>
<position x="-856.2311706542969" y="629.8186340332031" />
<id>9c0d14ff-0184-1000-ffff-ffff9e170ecc</id>
<name>PutElasticsearchRecord</name>
<position x="-856.0" y="632.0" />
<styles />
<comment />
<class>org.apache.nifi.processors.elasticsearch.PutElasticsearchHttpRecord</class>
<class>org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-elasticsearch-nar</artifact>
<artifact>nifi-elasticsearch-restapi-nar</artifact>
<version>1.12.1</version>
</bundle>
<maxConcurrentTasks>1</maxConcurrentTasks>
......@@ -9465,89 +9465,43 @@
<executionNode>ALL</executionNode>
<runDurationNanos>0</runDurationNanos>
<property>
<name>elasticsearch-http-url</name>
<value>${elastic_url}</value>
</property>
<property>
<name>SSL Context Service</name>
<value>83443c00-b286-366a-b8e0-2f51527ab8e5</value>
</property>
<property>
<name>Character Set</name>
<value>UTF-8</value>
</property>
<property>
<name>Username</name>
<value>${elastic_username}</value>
</property>
<property>
<name>Password</name>
<value>enc{907d07dbc9a93739b87296d143791fa32517e337630f9e98716909aae03b3cbd1e1c116bc9bbe793879e5d5d8d5b2724}</value>
</property>
<property>
<name>elasticsearch-http-connect-timeout</name>
<value>5 secs</value>
</property>
<property>
<name>elasticsearch-http-response-timeout</name>
<value>15 secs</value>
</property>
<property>
<name>proxy-configuration-service</name>
</property>
<property>
<name>elasticsearch-http-proxy-host</name>
</property>
<property>
<name>elasticsearch-http-proxy-port</name>
<name>el-rest-fetch-index</name>
<value>${data_index}-${now():format("yyyy-MM-dd")}</value>
</property>
<property>
<name>proxy-username</name>
<name>el-rest-type</name>
</property>
<property>
<name>proxy-password</name>
<name>el-rest-client-service</name>
<value>9c106232-0184-1000-ffff-fffff9d8684d</value>
</property>
<property>
<name>put-es-record-record-reader</name>
<name>put-es-record-reader</name>
<value>179dd31f-89ed-3179-adb2-85a9c61869ce</value>
</property>
<property>
<name>put-es-record-record-writer</name>
</property>
<property>
<name>put-es-record-log-all-errors</name>
<value>false</value>
<name>put-es-record-batch-size</name>
<value>100</value>
</property>
<property>
<name>put-es-record-id-path</name>
</property>
<property>
<name>put-es-record-index</name>
<value>${data_index}-${now():format("yyyy-MM-dd")}</value>
</property>
<property>
<name>put-es-record-type</name>
<value>_doc</value>
<name>put-es-record-index-record-path</name>
</property>
<property>
<name>put-es-record-index-op</name>
<value>index</value>
<name>put-es-record-type-record-path</name>
</property>
<property>
<name>suppress-nulls</name>
<value>always-suppress</value>
<name>put-es-record-log-error-responses</name>
<value>true</value>
</property>
<property>
<name>Date Format</name>
</property>
<property>
<name>Time Format</name>
</property>
<property>
<name>Timestamp Format</name>
<name>put-es-record-error-writer</name>
</property>
<autoTerminatedRelationship>success</autoTerminatedRelationship>
<autoTerminatedRelationship>failure</autoTerminatedRelationship>
<autoTerminatedRelationship>errors</autoTerminatedRelationship>
</processor>
<inputPort>
<id>e7d34e01-babe-3022-ad9b-a7620e7c0f38</id>
......@@ -9557,21 +9511,18 @@
<scheduledState>RUNNING</scheduledState>
</inputPort>
<connection>
<id>3280c550-2117-37a6-8b5e-3bc1953fa17e</id>
<id>9c0d2a48-0184-1000-ffff-ffffc4b80f44</id>
<name />
<bendPoints>
<bendPoint x="-393.2311706542969" y="669.8186340332031" />
<bendPoint x="-393.2311706542969" y="719.8186340332031" />
</bendPoints>
<bendPoints />
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</sourceId>
<sourceId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</sourceId>
<sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>9c0d14ff-0184-1000-ffff-ffff9e170ecc</destinationId>
<destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship>retry</relationship>
<relationship />
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
......@@ -9580,25 +9531,71 @@
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<connection>
<id>5de8f98f-ce46-3565-b0ce-7f8ecf518c53</id>
<id>9c0d9724-0184-1000-ffff-ffffe970e040</id>
<name />
<bendPoints />
<bendPoints>
<bendPoint x="-391.0" y="677.0" />
<bendPoint x="-368.0" y="720.0" />
</bendPoints>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<sourceId>e7d34e01-babe-3022-ad9b-a7620e7c0f38</sourceId>
<sourceId>9c0d14ff-0184-1000-ffff-ffff9e170ecc</sourceId>
<sourceGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>8b48f28f-2379-3f0f-81fe-4e1b93e72666</destinationId>
<sourceType>PROCESSOR</sourceType>
<destinationId>9c0d14ff-0184-1000-ffff-ffff9e170ecc</destinationId>
<destinationGroupId>7ebf304b-4978-3adc-ac31-470fb76e5029</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<relationship />
<relationship>retry</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>60 sec</flowFileExpiration>
<flowFileExpiration>0 sec</flowFileExpiration>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<partitioningAttribute />
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
</connection>
<controllerService>
<id>9c106232-0184-1000-ffff-fffff9d8684d</id>
<name>ElasticSearchClientService</name>
<comment />
<class>org.apache.nifi.elasticsearch.ElasticSearchClientServiceImpl</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-elasticsearch-client-service-nar</artifact>
<version>1.12.1</version>
</bundle>
<enabled>true</enabled>
<property>
<name>el-cs-http-hosts</name>
<value>${elastic_urls}</value>
</property>
<property>
<name>el-cs-username</name>
<value>admin</value>
</property>
<property>
<name>el-cs-password</name>
<value>enc{18f431a8121ccd08597b03628773453455e445222fe72b279f9f67a3f2775b5275c6752eb316a1521d7748b314b06932}</value>
</property>
<property>
<name>el-cs-ssl-context-service</name>
</property>
<property>
<name>el-cs-connect-timeout</name>
<value>5000</value>
</property>
<property>
<name>el-cs-socket-timeout</name>
<value>60000</value>
</property>
<property>
<name>el-cs-retry-timeout</name>
<value>60000</value>
</property>
<property>
<name>el-cs-charset</name>
<value>UTF-8</value>
</property>
</controllerService>
</processGroup>
<processGroup>
<id>2bb2f914-0172-1000-0000-0000240c76e4</id>
......@@ -11976,7 +11973,7 @@
</property>
<property>
<name>Truststore Password</name>
<value>enc{4e264a7abbb4164f1c4fac0757c23fe630b181bc223b6c985c7de2c09ef870dc}</value>
<value>enc{b061a02ce6812799b478388abce887518f82325e7db568bd814dc716b1ffb35e}</value>
</property>
<property>
<name>Truststore Type</name>
......@@ -12616,9 +12613,9 @@
<variable name="misp_first_interval" value="60d" />
<variable name="elastic_username" value="{{ elastic_username }}" />
<variable name="misp_url" value="{{ misp_url }}" />
<variable name="elastic_url" value="https://{{ soctoolsproxy }}:9200" />
<variable name="elastic_urls" value="https://soctools-opensearch-1:9200/,https://soctools-opensearch-2:9200/" />
<variable name="elastic_password" value="{{lookup('password', '{{playbook_dir}}/secrets/passwords/opensearches_adminpass')}}" />
</rootGroup>
<controllerServices />
<reportingTasks />
</flowController>
</flowController>
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment