Can't send data to elastic after upgrade the version

Hi, previously we used Elastic & Logstash version 7.15 to store data. But after we upgraded to version 8 the data could not be sent. We used the same Logstash configuration and added a few things that changed in version 8 in the output section.

Here's the Logstash configuration:

> input {
>   file {
>     path => "*.log"
>     start_position => "beginning"
>     codec => "line"
>     mode => "read"
>     file_completed_action => "delete"
>     sincedb_path => "/dev/null"
>   }
> }
> 
> filter {
>     mutate { 
> 	  rename => ["message", "rawMessage" ]
> 	}
> 
> 	
> 	mutate {
>       gsub => ["rawMessage", ";", ""]
> 	}
> 	
> 	
>     grok {
> 	  break_on_match => false
> 	  match => { "rawMessage" => [
>                             "(?<MTI>(MTI :)([a-zA-Z0-9_]+)*)",
> 							"(?<Response_Code>(Response.Code.:)([^|]*))",
> 							"(?<Hostname>(hostname.:)([^|]*))",
> 							"(?<Filename>(filename.:)([^|]*))",
> 							"(?<date>(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|)))"
> 							] 
> 				}
>     }
> 	
>     date {
>       match => [ "date", "yyyy-MM-dd'T'HH:mm:ss.SSSS"]
> 	  target=> "@timestamp"
>     }
> 	
> 	mutate {
> 	  gsub => ["MTI", "MTI :", ""]
> 	  gsub => ["Response_Code", "Response Code :", ""]
> 	  gsub => ["Hostname", "hostname :", ""]
> 	  gsub => ["Filename", "filename :", ""]
> 	}
> 	
> 	mutate {
> 	  gsub => ["rawMessage", " date :%{date}", ""]
> 	}
> 	
> 
> 	
> 	if "Response Code" in [rawMessage]
> 			{
> 				mutate {
> 					add_field => { "LogLevel" => "ERROR" }
> 					
> 				}
> 			}
> 	
> 	if "Smart" in [path]
> 			{
> 				mutate {
> 					add_field => { "App_Log" => "Smart" }
> 					
> 				}
> 			}
> 	else if "server1" in [path]
> 			{
> 				mutate {
> 					add_field => { "App_Log" => "server1" }
> 					
> 				}
> 			}
> }
> output {
> 	if "data1" in [path]
> 	{
> 		elasticsearch { 
> 		hosts => ["https://192.xxx.xx.x:9200"]
> 		index => "template-index-1"
> 	        user => "admin"
> 		password => "admin"
> 		}
> 	}
> }

The log:

MTI :0210 (Financial) | Primary Account Number :4043052015067244 | Processing Code :391000 (Transfer Inquiry from Saving Account to Unspecified account) | Amount, Transaction: 102000,00 | Delivery Channel:ATM | Surcharge Fee :39611111 | Retrieval Reference Number :002929155716 | Response Code :53 (Saving Account not Found) | Primary Bit Maps :F33840098A808002 | Secondary Bit Map :0000000002000006 | Card Holder Billing Fee :00000000 | Date - Local Transaction :1026 | Time - Local Transaction:171747 | Transmission Date and Time:26-10 10:17:47 | System Trace Audit Number :155716 | Acquiring Institution ID :00000000008 | Forwarding Institution ID :450005 | Card Acceptor Terminal ID :S1AW13FS | Card Acceptor Merchant ID :- | Card Acceptor Name/Location :- | Private - Additional Data :- | Transaction Currency Code :360 | Additional Amounts :- | Private - Service Code :0900 | Network Management Information Code :- | Original Data Element :- | Account Identification 1 - Source Account :- | Account Identification 2 - Beneficiary Account:600085755826730 | Transaction Description:- | Private - Transfer Action Code:- | Private - Source Institution Code:451 | Private - Beneficiary Institution Code:451 | hostname :CBS-QUERY-AP1 | filename :tcserver2 | date :2022-10-26T10:18:20.6850;

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.