Grok not parsit

Elastic Stack Logstash
1

I have PostgreSQL logs in stderr format (csv format is not possible to do).

The message does not fall into the grok. I think that will fall into the last rule where the tag "foo5"

2019-04-01 06:59:22 EEST [3103294]: db=payments,user=24pay,app=24pay@erlang-payment-app-ua-1,client=10.61.147.181,0,BIND LOG: duration: 0.115 ms bind call_func_prepared_3793fa2eca1b6c75c29e8ac5974e40df: select get_archive3(in_corp_eid := $1, in_offset := $2, in_limit := $3, in_pmnt_statuses := $4, in_start_service_update_time := $5, in_end_service_update_time := $6, in_user_sign_level := $7, in_need_pmnts_for_sign := $8, in_internal_types := $9);

input {
         file {
    	mode => "read"
    	path => [ "/DATA/tmp/2.txt" ]
    	start_position => "beginning"

	codec => multiline {
	    pattern => "^%{TIMESTAMP_ISO8601}"
	    negate => true
	    what => "previous"
	}
    }
}

filter {

    grok {
	    patterns_dir => [ "/DATA/logstash-6.7.0/config/patterns" ]
	    match => { "message" => "%{TIMESTAMP_ISO8601:date} %{DATA}db=%{DATA:db},user=%{DATA:username},app=%{GREEDYDATA:app},client=%{IPV4:client},%{NUMBER:transaction_id},%{GREEDYDATA:command_tag}:%{GREEDYDATA} duration: %{BASE16FLOAT:duration} %{GREEDYDATA}: %{GREEDYDATA:query} --%{REQUESTID:requestid}" }
	    add_tag => [ "foo1" ]
    }

    if "_grokparsefailure" in [tags] {
        grok {
	    match => { "message" => "%{TIMESTAMP_ISO8601:date} %{DATA}db=%{DATA:db},user=%{DATA:username},app=%{GREEDYDATA:app},client=%{IPV4:client},%{NUMBER:transaction_id},%{GREEDYDATA:command_tag}:%{GREEDYDATA} duration: %{BASE16FLOAT:duration} %{GREEDYDATA}Query Text: %{GREEDYDATA:query}" }
	    add_tag => [ "foo2" ]
	}
    }
    else if "_grokparsefailure" in [tags] {
        grok {
	    patterns_dir => [ "/DATA/logstash-6.7.0/config/patterns" ]
	    match => { "message" => "%{TIMESTAMP_ISO8601:date} %{DATA}db=%{DATA:db},user=%{DATA:username},app=%{GREEDYDATA:app},client=%{IPV4:client},%{GREEDYDATA}duration: %{BASE16FLOAT:duration}%{GREEDYDATA}: %{GREEDYDATA:query1}%{REQUESTID:requestid}%{GREEDYDATA:query2}" }
	    add_field => ["query", "%{query1} %{requestid} %{query2}" ]
	    add_tag => [ "foo3" ]
	}
    }
    else if "_grokparsefailure" in [tags] {
	grok {
	    match => { "message" => "%{TIMESTAMP_ISO8601:date} %{DATA}db=%{DATA:db},user=%{DATA:username},app=%{GREEDYDATA:app},client=%{IPV4:client},%{NUMBER:transaction_id},%{GREEDYDATA:command_tag}:%{SPACE}duration: %{BASE16FLOAT:duration}%{GREEDYDATA}: %{GREEDYDATA:query}" }
	    add_tag => [ "foo4" ]
	}
    }
    else if "_grokparsefailure" in [tags] {
        grok {
	    match => { "message" => "%{TIMESTAMP_ISO8601:date} %{DATA}db=%{DATA:db},user=%{DATA:username},app=%{GREEDYDATA:app},client=%{IPV4:client},%{NUMBER:transaction_id},%{GREEDYDATA:command_tag}:%{SPACE} %{GREEDYDATA:query}" }
	    add_tag => [ "foo5" ]
	}
    }

}

output {
    jdbc {
	connection_string => 'jdbc:postgresql://192.168.1.32:5432/database'
	username => 'login'
	password => 'password'
	statement => [ "INSERT INTO public.postgres_log_test(log_time, database_name, user_name, application_name, client, transaction_id, command_tag, duration, query, requestid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", date, db, username, app, client, transaction_id, command_tag, duration, query, requestid ]    }

stdout {}

}

Result

{
"path" => "/DATA/tmp/2.txt",
"tags" => [
[0] "_grokparsefailure"
],
"message" => "2019-04-01 06:59:22 EEST [3103294]: db=payments,user=24pay,app=24pay@erlang-payment-app-ua-1,client=10.61.147.181,0,BIND LOG: duration: 0.115 ms bind call_func_prepared_3793fa2eca1b6c75c29e8ac5974e40df: select get_archive3(in_corp_eid := $1, in_offset := $2, in_limit := $3, in_pmnt_statuses := $4, in_start_service_update_time := $5, in_end_service_update_time := $6, in_user_sign_level := $7, in_need_pmnts_for_sign := $8, in_internal_types := $9);",
"host" => "vm-the-punisher.ipa.pbank.com.ua",
"@timestamp" => 2019-04-18T10:14:06.545Z,
"@version" => "1"
}

Why does the message not go into the grok?

2

Hard to say. What does your REQUESTID pattern look like? For me I get a match on foo4

{
            "db" => "payments",
       "message" => "2019-04-01 06:59:22 EEST [3103294]: db=payments,user=24pay,app=24pay@erlang-payment-app-ua-1,client=10.61.147.181,0,BIND LOG: duration: 0.115 ms bind call_func_prepared_3793fa2eca1b6c75c29e8ac5974e40df: select get_archive3(in_corp_eid := $1, in_offset := $2, in_limit := $3, in_pmnt_statuses := $4, in_start_service_update_time := $5, in_end_service_update_time := $6, in_user_sign_level := $7, in_need_pmnts_for_sign := $8, in_internal_types := $9);",
   "command_tag" => "BIND LOG",
          "tags" => [
    [0] "_grokparsefailure",
    [1] "foo4"
],
      "duration" => "0.115",
           "app" => "24pay@erlang-payment-app-ua-1",
         "query" => "select get_archive3(in_corp_eid := $1, in_offset := $2, in_limit := $3, in_pmnt_statuses := $4, in_start_service_update_time := $5, in_end_service_update_time := $6, in_user_sign_level := $7, in_need_pmnts_for_sign := $8, in_internal_types := $9);",
"transaction_id" => "0",
      "username" => "24pay",
          "date" => "2019-04-01 06:59:22",
        "client" => "10.61.147.181",
}
3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.