Skip to content
Snippets Groups Projects
Commit d1559b4f authored by Václav Bartoš's avatar Václav Bartoš
Browse files

fixed processing of filebeat data 

related to migration from elasticsearch to opensearch
parent cafb8d1f
No related branches found
No related tags found
1 merge request!5Migrate from ODFE to OpenSearch
Hide whitespace changes
Inline Side-by-side
inventories/filebeat
+ 3
− 3
View file @ d1559b4f
......@@ -3,9 +3,9 @@ soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
soctools-opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/opensearch-dashboards_stdout.log"]' FILEBEAT_LOG_TYPE="opensearch-dashboards" FILEBEAT_LOG_FORMAT="json"
soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/opt/opensearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="opensearch" FILEBEAT_LOG_FORMAT="json"
soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/opt/opensearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="opensearch" FILEBEAT_LOG_FORMAT="json"
soctools-opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/opensearch-dashboards_stdout.log"]' FILEBEAT_LOG_TYPE="osdashboards" FILEBEAT_LOG_FORMAT="json"
soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log","/var/opt/rh/rh-mariadb103/lib/mysql/server_audit.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"
......
roles/nifi/templates/flow.xml.j2
+ 31
− 28
View file @ d1559b4f
......@@ -13,7 +13,7 @@
<processGroup>
<id>41088add-955b-3611-a0de-2c18b79b678c</id>
<name>Data processing</name>
<position x="1216.0" y="256.0" />
<position x="1216.0" y="264.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
......@@ -3862,21 +3862,21 @@
<processGroup>
<id>870d6d68-7a0a-3505-8c42-0d6064fe43f6</id>
<name>Data input</name>
<position x="830.4597621124223" y="407.3463126314215" />
<position x="832.0" y="408.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<outputPort>
<id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
<name>To enrichment</name>
<position x="168.0" y="616.0" />
<id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
<name>To data output</name>
<position x="-840.0" y="512.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
<outputPort>
<id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
<name>To data output</name>
<position x="-840.0" y="512.0" />
<id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
<name>To enrichment</name>
<position x="168.0" y="616.0" />
<comments />
<scheduledState>RUNNING</scheduledState>
</outputPort>
......@@ -3895,16 +3895,16 @@
<scheduledState>STOPPED</scheduledState>
</inputPort>
<outputPort>
<id>27d5761b-0172-1000-0000-000059275dad</id>
<name>To enrichment</name>
<position x="-312.0" y="328.0" />
<id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
<name>To data output</name>
<position x="-632.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
<outputPort>
<id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
<name>To data output</name>
<position x="-632.0" y="328.0" />
<id>27d5761b-0172-1000-0000-000059275dad</id>
<name>To enrichment</name>
<position x="-312.0" y="328.0" />
<comments />
<scheduledState>STOPPED</scheduledState>
</outputPort>
......@@ -4331,12 +4331,8 @@
<value>${log_type:equals("keycloak")}</value>
</property>
<property>
<name>kibana</name>
<value>${log_type:equals("kibana")}</value>
</property>
<property>
<name>elasticsearch</name>
<value>${log_type:equals("elasticsearch")}</value>
<name>osdashboards</name>
<value>${log_type:equals("opensearch-dashboards")}</value>
</property>
<property>
<name>suricata</name>
......@@ -4362,6 +4358,10 @@
<name>nifi</name>
<value>${log_type:equals("nifi")}</value>
</property>
<property>
<name>opensearch</name>
<value>${log_type:equals("opensearch")}</value>
</property>
<property>
<name>zookeeper</name>
<value>${log_type:equals("zookeeper")}</value>
......@@ -7528,7 +7528,7 @@
</processGroup>
<processGroup>
<id>7263390f-914c-1f6e-9451-75f908ed8816</id>
<name>Elasticsearch</name>
<name>OpenSearch</name>
<position x="-1904.0" y="488.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
......@@ -7571,7 +7571,7 @@
</property>
<property>
<name>data_index</name>
<value>logs-elasticsearch</value>
<value>logs-opensearch</value>
</property>
</processor>
<inputPort>
......@@ -8393,7 +8393,7 @@
</processGroup>
<processGroup>
<id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
<name>Kibana</name>
<name>Opensearch Dashboards</name>
<position x="-440.0" y="864.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
......@@ -8436,7 +8436,7 @@
</property>
<property>
<name>data_index</name>
<value>logs-kibana</value>
<value>logs-osdashboards</value>
</property>
</processor>
<inputPort>
......@@ -8666,7 +8666,7 @@
<destinationId>39ce3238-1ebd-1c2c-b724-01d18f147b6f</destinationId>
<destinationGroupId>7263390f-914c-1f6e-9451-75f908ed8816</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<relationship>elasticsearch</relationship>
<relationship>opensearch</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
......@@ -8825,9 +8825,12 @@
<id>6196cd03-0176-1000-ffff-ffffd39b8c82</id>
<name />
<bendPoints>
<bendPoint x="-758.2385864257812" y="580.6355590820312" />
<bendPoint x="-740.111083984375" y="604.8054809570312" />
<bendPoint x="-738.6004638671875" y="609.3373413085938" />
<bendPoint x="-576.0" y="896.0" />
</bendPoints>
<labelIndex>0</labelIndex>
<labelIndex>3</labelIndex>
<zIndex>0</zIndex>
<sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
<sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
......@@ -8835,7 +8838,7 @@
<destinationId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</destinationId>
<destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<relationship>kibana</relationship>
<relationship>osdashboards</relationship>
<maxWorkQueueSize>10000</maxWorkQueueSize>
<maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
<flowFileExpiration>0 sec</flowFileExpiration>
......@@ -9424,7 +9427,7 @@
<processGroup>
<id>e9c19adc-c8a4-327e-ad24-24e71fd3474e</id>
<name>Data output</name>
<position x="829.4446253936723" y="1015.2711478364996" />
<position x="848.0" y="992.0" />
<comment />
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment