fixed processing of filebeat data

related to migration from elasticsearch to opensearch

inventories/filebeat

@@ -3,9 +3,9 @@ soctools-nifi-1 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-curre
 soctools-nifi-2 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-nifi-3 ansible_connection=docker FILEBEAT_FILES='["/opt/nifi/nifi-current/logs/nifi-app.log","/opt/nifi/nifi-current/logs/nifi-bootstrap.log","/opt/nifi/nifi-current/logs/nifi-user.log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="nifi" FILEBEAT_LOG_FORMAT="text"
 soctools-misp ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-php72/log/php-fpm/*.log","/var/opt/rh/rh-redis32/log/redis/redis.log","/var/log/httpd/*log","/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="misp" FILEBEAT_LOG_FORMAT="text"
-soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
-soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/usr/share/elasticsearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="elasticsearch" FILEBEAT_LOG_FORMAT="json"
-soctools-opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/opensearch-dashboards_stdout.log"]' FILEBEAT_LOG_TYPE="opensearch-dashboards" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-1 ansible_connection=docker FILEBEAT_FILES='["/opt/opensearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="opensearch" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-2 ansible_connection=docker FILEBEAT_FILES='["/opt/opensearch/logs/soctools-cluster_server.json"]' FILEBEAT_LOG_TYPE="opensearch" FILEBEAT_LOG_FORMAT="json"
+soctools-opensearch-dashboards ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/opensearch-dashboards_stdout.log"]' FILEBEAT_LOG_TYPE="osdashboards" FILEBEAT_LOG_FORMAT="json"
 soctools-keycloak ansible_connection=docker FILEBEAT_FILES='["/var/log/supervisor/*.log"]' FILEBEAT_LOG_TYPE="keycloak" FILEBEAT_LOG_FORMAT="json"
 soctools-mysql ansible_connection=docker FILEBEAT_FILES='["/var/opt/rh/rh-mariadb103/log/mariadb/mariadb.log","/var/opt/rh/rh-mariadb103/lib/mysql/server_audit.log"]' FILEBEAT_LOG_TYPE="mysql" FILEBEAT_LOG_FORMAT="text"
 soctools-haproxy ansible_connection=docker FILEBEAT_SYSLOG_PORT=9000 FILEBEAT_LOG_TYPE="haproxy" FILEBEAT_LOG_FORMAT="text"

roles/nifi/templates/flow.xml.j2

@@ -13,7 +13,7 @@
     <processGroup>
       <id>41088add-955b-3611-a0de-2c18b79b678c</id>
       <name>Data processing</name>
-      <position x="1216.0" y="256.0" />
+      <position x="1216.0" y="264.0" />
       <comment />
       <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
       <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
@@ -3862,21 +3862,21 @@
       <processGroup>
         <id>870d6d68-7a0a-3505-8c42-0d6064fe43f6</id>
         <name>Data input</name>
-        <position x="830.4597621124223" y="407.3463126314215" />
+        <position x="832.0" y="408.0" />
         <comment />
         <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
         <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
         <outputPort>
-          <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
-          <name>To enrichment</name>
-          <position x="168.0" y="616.0" />
+          <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
+          <name>To data output</name>
+          <position x="-840.0" y="512.0" />
           <comments />
           <scheduledState>RUNNING</scheduledState>
         </outputPort>
         <outputPort>
-          <id>21a9e277-2d80-359a-9c57-cb76d8962e6d</id>
-          <name>To data output</name>
-          <position x="-840.0" y="512.0" />
+          <id>20b01ab3-3a8d-3573-b95d-a4a45494050f</id>
+          <name>To enrichment</name>
+          <position x="168.0" y="616.0" />
           <comments />
           <scheduledState>RUNNING</scheduledState>
         </outputPort>
@@ -3895,16 +3895,16 @@
             <scheduledState>STOPPED</scheduledState>
           </inputPort>
           <outputPort>
-            <id>27d5761b-0172-1000-0000-000059275dad</id>
-            <name>To enrichment</name>
-            <position x="-312.0" y="328.0" />
+            <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
+            <name>To data output</name>
+            <position x="-632.0" y="328.0" />
             <comments />
             <scheduledState>STOPPED</scheduledState>
           </outputPort>
           <outputPort>
-            <id>27d5dab2-0172-1000-ffff-ffffab5c50be</id>
-            <name>To data output</name>
-            <position x="-632.0" y="328.0" />
+            <id>27d5761b-0172-1000-0000-000059275dad</id>
+            <name>To enrichment</name>
+            <position x="-312.0" y="328.0" />
             <comments />
             <scheduledState>STOPPED</scheduledState>
           </outputPort>
@@ -4331,12 +4331,8 @@
               <value>${log_type:equals("keycloak")}</value>
             </property>
             <property>
-              <name>kibana</name>
-              <value>${log_type:equals("kibana")}</value>
-            </property>
-            <property>
-              <name>elasticsearch</name>
-              <value>${log_type:equals("elasticsearch")}</value>
+              <name>osdashboards</name>
+              <value>${log_type:equals("opensearch-dashboards")}</value>
             </property>
             <property>
               <name>suricata</name>
@@ -4362,6 +4358,10 @@
               <name>nifi</name>
               <value>${log_type:equals("nifi")}</value>
             </property>
+            <property>
+              <name>opensearch</name>
+              <value>${log_type:equals("opensearch")}</value>
+            </property>
             <property>
               <name>zookeeper</name>
               <value>${log_type:equals("zookeeper")}</value>
@@ -7528,7 +7528,7 @@
           </processGroup>
           <processGroup>
             <id>7263390f-914c-1f6e-9451-75f908ed8816</id>
-            <name>Elasticsearch</name>
+            <name>OpenSearch</name>
             <position x="-1904.0" y="488.0" />
             <comment />
             <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
@@ -7571,7 +7571,7 @@
               </property>
               <property>
                 <name>data_index</name>
-                <value>logs-elasticsearch</value>
+                <value>logs-opensearch</value>
               </property>
             </processor>
             <inputPort>
@@ -8393,7 +8393,7 @@
           </processGroup>
           <processGroup>
             <id>f0f934a9-853a-1a19-a9cc-f878a5606bce</id>
-            <name>Kibana</name>
+            <name>Opensearch Dashboards</name>
             <position x="-440.0" y="864.0" />
             <comment />
             <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
@@ -8436,7 +8436,7 @@
               </property>
               <property>
                 <name>data_index</name>
-                <value>logs-kibana</value>
+                <value>logs-osdashboards</value>
               </property>
             </processor>
             <inputPort>
@@ -8666,7 +8666,7 @@
             <destinationId>39ce3238-1ebd-1c2c-b724-01d18f147b6f</destinationId>
             <destinationGroupId>7263390f-914c-1f6e-9451-75f908ed8816</destinationGroupId>
             <destinationType>INPUT_PORT</destinationType>
-            <relationship>elasticsearch</relationship>
+            <relationship>opensearch</relationship>
             <maxWorkQueueSize>10000</maxWorkQueueSize>
             <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
             <flowFileExpiration>0 sec</flowFileExpiration>
@@ -8825,9 +8825,12 @@
             <id>6196cd03-0176-1000-ffff-ffffd39b8c82</id>
             <name />
             <bendPoints>
+              <bendPoint x="-758.2385864257812" y="580.6355590820312" />
+              <bendPoint x="-740.111083984375" y="604.8054809570312" />
+              <bendPoint x="-738.6004638671875" y="609.3373413085938" />
               <bendPoint x="-576.0" y="896.0" />
             </bendPoints>
-            <labelIndex>0</labelIndex>
+            <labelIndex>3</labelIndex>
             <zIndex>0</zIndex>
             <sourceId>8962ad5a-0175-1000-ffff-ffffde6db5a6</sourceId>
             <sourceGroupId>0c790562-0175-1000-ffff-ffffeaaeafc3</sourceGroupId>
@@ -8835,7 +8838,7 @@
             <destinationId>a22b30c4-53f8-19c0-bdbb-0632e99a17d9</destinationId>
             <destinationGroupId>f0f934a9-853a-1a19-a9cc-f878a5606bce</destinationGroupId>
             <destinationType>INPUT_PORT</destinationType>
-            <relationship>kibana</relationship>
+            <relationship>osdashboards</relationship>
             <maxWorkQueueSize>10000</maxWorkQueueSize>
             <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize>
             <flowFileExpiration>0 sec</flowFileExpiration>
@@ -9424,7 +9427,7 @@
       <processGroup>
         <id>e9c19adc-c8a4-327e-ad24-24e71fd3474e</id>
         <name>Data output</name>
-        <position x="829.4446253936723" y="1015.2711478364996" />
+        <position x="848.0" y="992.0" />
         <comment />
         <flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
         <flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>