RESOLVED: Select a grok pattern and add “my_body” field

Elastic Stack Logstash
1

I want to be able to select a grok pattern and add “my_body” field to it to force an action later.
I can get it to add multiple tags or no tags, but not a single tag.
Below is the latest filter version of many iterations, but I can never get it to work as above.
I have tried:
if "_grokparsefailure" in [tabs] {
if !( [tabs] =~ “grok27” ){
etc.

${LOGSTASH_DIR}/bin/logstash --version
logstash 2.3.1

Test Events:
dplrgid8@:/home/dplrgid8/logstash/dev/templates/$ cat /tmp/out
[15 Dec 2016, 09:01:18 ] ERROR : com.sun.tools.attach.AttachNotSupportedException: Unable to open socket file: target process not responding or HotSpot VM not loaded
at sun.tools.attach.LinuxVirtualMachine.(LinuxVirtualMachine.java:106)
at sun.tools.attach.LinuxAttachProvider.attachVirtualMachine(LinuxAttachProvider.java:78)
at com.sun.tools.attach.VirtualMachine.attach(VirtualMachine.java:255)
at com.globalids.env.SystemHealthMonitor.(SystemHealthMonitor.java:131)
at com.globalids.env.SystemHealthMonitor$PeriodicTask.run(SystemHealthMonitor.java:761)
at java.util.TimerThread.mainLoop(Timer.java:555)
at java.util.TimerThread.run(Timer.java:505)

Filter shm.exception:
dplrgid8@:/home/dplrgid8/logstash/dev/bin/$ cat ../templates/shm.exception.filter.template

    if [type] == "MSG_TYPE" {

            grok {
                    match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}:%{SPACE}%{NOTSPACE:err_num}:%{SPACE}%{GREEDYDATA:narrative}" }
                    remove_tag => ["_grokparsefailure"]
                    add_field => { "tags" => "grok2701" }
            }

            if "_grokparsefailure" in [tabs] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}Index:%{SPACE}%{NOTSPACE:index},%{SPACE}Size:%{SPACE}%{NOTSPACE:size}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { "tags" => "grok2702" }
                    }
            }

            else if "_grokparsefailure" in [tabs] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}%{INT:err_num}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { "tags" => "grok2703" }
                    }
            }


            else if "_grokparsefailure" in [tabs] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { "tags" => "grok2706" }
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

Test Below

            else if "_grokparsefailure" in [tabs] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { "tags" => "grok2707" }
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

TEST Below

            else if "_grokparsefailure" in [tabs] {
                    mutate {
                            add_field => { "tags" => "grok2799" }
                    }
            }
    }

Results:
Settings: Default pipeline workers: 8
Pipeline main started
{
"@timestamp" => "2016-12-15T21:18:15.934Z",
"message" => "[15 Dec 2016, 09:01:18 ] ERROR : com.sun.tools.attach.AttachNotSupportedException: Unable to open socket file: target process not responding or HotSpot VM not loaded\n\tat sun.tools.attach.LinuxVirtualMachine.(LinuxVirtualMachine.java:106)\n\tat sun.tools.attach.LinuxAttachProvider.attachVirtualMachine(LinuxAttachProvider.java:78)\n\tat com.sun.tools.attach.VirtualMachine.attach(VirtualMachine.java:255)\n\tat com.globalids.env.SystemHealthMonitor.(SystemHealthMonitor.java:131)\n\tat com.globalids.env.SystemHealthMonitor$PeriodicTask.run(SystemHealthMonitor.java:761)\n\tat java.util.TimerThread.mainLoop(Timer.java:555)\n\tat java.util.TimerThread.run(Timer.java:505)",
"@version" => "1",
"tags" => [
[0] "multiline",
[1] "_grokparsefailure"
],
"path" => "/opt/app/globalid/data/server/Release-9-0/logs/shm_exception.log",
"host" => "hltv0777.hydc.sbc.com",
"type" => "shm.exception",
"application" => "DPLR_GID",
"environment" => "qa",
"component" => "CHILD",
"logstash_version" => "2.3.1",
"filter_date" => "2016/12/15 15:17:52 CST"
}

2

The first thing that stands out to me is that you are using a [tabs] field in your conditionals to check for the "_grokparsefailure" tag, shouldn't this be in [tags]?

Also when you do the mutate, you should use the logstash add_tag setting:

 else if "_grokparsefailure" in [tabs] {
   mutate {
      add_tag => [ "grok2799" ]
   }
 }
3

Troy,
Thank you for the prompt reply and for correcting my typo.
I have really been working on this too long.
I corrected the typo with the same results:

Settings: Default pipeline workers: 8
Pipeline main started
{
"@timestamp" => "2016-12-18T00:33:57.712Z",
"message" => "[15 Dec 2016, 09:01:18 ] ERROR : com.sun.tools.attach.AttachNotSupportedException: Unable to open socket file: target process not responding or HotSpot VM not loaded\n\tat sun.tools.attach.LinuxVirtualMachine.(LinuxVirtualMachine.java:106)\n\tat sun.tools.attach.LinuxAttachProvider.attachVirtualMachine(LinuxAttachProvider.java:78)\n\tat com.sun.tools.attach.VirtualMachine.attach(VirtualMachine.java:255)\n\tat com.globalids.env.SystemHealthMonitor.(SystemHealthMonitor.java:131)\n\tat com.globalids.env.SystemHealthMonitor$PeriodicTask.run(SystemHealthMonitor.java:761)\n\tat java.util.TimerThread.mainLoop(Timer.java:555)\n\tat java.util.TimerThread.run(Timer.java:505)",
"@version" => "1",
"tags" => [
[0] "multiline",
[1] "_grokparsefailure"
],
"path" => "/opt/app/globalid/data/server/Release-9-0/logs/shm_exception.log",
"host" => "hltv0777.hydc.sbc.com",
"type" => "shm.exception",
"application" => "DPLR_GID",
"environment" => "qa",
"component" => "CHILD",
"logstash_version" => "2.3.1",
"filter_date" => "2016/12/17 18:33:32 CST"
}

Attempting to change mutate to this resulted in a configtest failure: \telse if "_grokparsefailure" in [tags] {\n\t\t\tmutate {\n\t\t\t\tadd_tags => { "grok2799" ", :level=>:error}

else if "_grokparsefailure" in [tags] {
mutate {
add_tag => [ "grok2799" ]
}
}

Here is my code as it stands:
cat shm.exception.filter.template

    if [type] == "MSG_TYPE" {

            grok {
                    match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}:%{SPACE}%{NOTSPACE:err_num}:%{SPACE}%{GREEDYDATA:narrative}" }
                    remove_tag => ["_grokparsefailure"]
                    add_field => { tags =>  "grok2701" }
            }

            if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}Index:%{SPACE}%{NOTSPACE:index},%{SPACE}Size:%{SPACE}%{NOTSPACE:size}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2702" }
                    }
            }

            else if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}%{INT:err_num}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2703" }
                    }
            }


            else if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2706" }
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

            else if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2707" }
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

            else if "_grokparsefailure" in [tags] {
                    mutate {
                            add_field => { tags =>  "grok2799" }
                    }
            }
    }

Brian

4

Attempting to change mutate to this resulted in a configtest failure: \telse if "grokparsefailure" in [tags] {\n\t\t\tmutate {\n\t\t\t\taddtags => { "grok2799" ", :level=>:error}

The above error indicates you had a typo in one of your mutate sections and had

mutate {
  addtags => "grok2799"
}

This is incorrect and should throw an error. Please review my earlier response and see if you can identify the issue. When trying to add tags, you should not be using the standard add_field directive and instead use the add_tag directive.

After comparing the incorrect mutate section of this post, and the correct one in the earlier post, if you still do not understand what is wrong reply here and I will be glad to go into more detail.

EDIT: If you do need additional help, please post your config with all of the mutate sections updated with the add_tag directive. This will make it easier to assist.

5

Troy,
Thank you for your patience. I have corrected the issue with "mutate."
Below is the complete logstash.conf configuration:

input {
file {
type => "shm.exception"
path => [ "/opt/app/globalid/data/server/Release-9-0/logs/shm_exception.log" ]
start_position => "end"
codec => multiline {
pattern => '^\t|^\s|^FROM|^WHERE|^GROUP|^java.lang.NullPointerException|^Caused by:|^;|^-<|</|^/|^AUTH>|^Proxying request:|^TypeError:|^User auth failed:|^(Headers):|^topic"|Full thread dump|SVR4 Error|Heap|JNI |^"'
what => "previous"
}
add_field => {
"application" => "DPLR_GID"
"environment" => "qa"
"component" => "CHILD"
}
}
}

filter {
if( [message] !~ /(.+)/ ){
drop { }
}

    mutate {
            add_field => {
                    "logstash_version" => "2.3.1"
            }

            add_field => {
                    "filter_date" => "2016/12/22 09:06:33 CST"
            }
    }

    if [type] == "shm.exception" {

            grok {
                    match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}:%{SPACE}%{NOTSPACE:err_num}:%{SPACE}%{GREEDYDATA:narrative}" }
                    remove_tag => ["_grokparsefailure"]
                    add_field => { tags =>  "grok2701" }
            }

            if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}Index:%{SPACE}%{NOTSPACE:index},%{SPACE}Size:%{SPACE}%{NOTSPACE:size}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2702" }
                    }
            }

            else if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}%{INT:err_num}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2703" }
                    }
            }


            else if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2706" }
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

            else if "_grokparsefailure" in [tags] {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_field => { tags =>  "grok2707" }
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

            else if "_grokparsefailure" in [tags] {
                    mutate {
                            add_tag => [ "grok2799" ]
                    }
            }
    }

}
output {
stdout {
codec => rubydebug
}
}

The mutate issue is resolved.
There is still an issue with if "_grokparsefailure" in [tags] {. The test results are:

Settings: Default pipeline workers: 8
Pipeline main started
{
"@timestamp" => "2016-12-22T15:07:30.284Z",
"message" => "[22 Dec 2016, 06:50:39 ] ERROR : com.sun.tools.attach.AttachNotSupportedException: Unable to open socket file: target process not responding or HotSpot VM not loaded\n\tat sun.tools.attach.LinuxVirtualMachine.(LinuxVirtualMachine.java:106)\n\tat sun.tools.attach.LinuxAttachProvider.attachVirtualMachine(LinuxAttachProvider.java:78)\n\tat com.sun.tools.attach.VirtualMachine.attach(VirtualMachine.java:255)\n\tat java.lang.invoke.MethodHandle.invokeExact(MethodHandle.java)\n\tat com.globalids.env.SystemHealthMonitor.(SystemHealthMonitor.java:131)\n\tat com.globalids.env.SystemHealthMonitor$PeriodicTask.run(SystemHealthMonitor.java:761)\n\tat java.util.TimerThread.mainLoop(Timer.java:555)\n\tat java.util.TimerThread.run(Timer.java:505)",
"@version" => "1",
"tags" => [
[0] "multiline",
[1] "_grokparsefailure"
],
"path" => "/opt/app/globalid/data/server/Release-9-0/logs/shm_exception.log",
"host" => "hltv0777.hydc.sbc.com",
"type" => "shm.exception",
"application" => "DPLR_GID",
"environment" => "qa",
"component" => "CHILD",
"logstash_version" => "2.3.1",
"filter_date" => "2016/12/22 09:06:33 CST"
}

There may be a better way to do this. The point is to identify each grok pattern with a tag for troubleshooting purposes.
Brian

6

We can close this issue.
The magic was in removing the "else" part of "else if". It was causing the rule to terminate prematurely.

    if [type] == "shm.exception" {

            grok {
                    match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}:%{SPACE}%{NOTSPACE:err_num}:%{SPACE}%{GREEDYDATA:narrative}" }
                    remove_tag => ["_grokparsefailure"]
                    add_tag => [ "grok2701" ]
            }

            if( "_grokparsefailure" in [tags] ) {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}Index:%{SPACE}%{NOTSPACE:index},%{SPACE}Size:%{SPACE}%{NOTSPACE:size}" }
                            remove_tag => ["_grokparsefailure"]
                            add_tag => [ "grok2702" ]
                    }
            }

            if( "_grokparsefailure" in [tags] ) {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{NOTSPACE:msg_level}%{SPACE}:%{SPACE}%{INT:err_num}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_tag => [ "grok2703" ]
                    }
            }

            if( "_grokparsefailure" in [tags] ) {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_tag => [ "grok2706" ]
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

            if( "_grokparsefailure" in [tags] ) {
                    grok {
                            match => { "message" => "^\[%{MONTHDAY}%{SPACE}%{MONTH}%{SPACE}%{YEAR},%{SPACE}%{TIME}%{SPACE}\]%{SPACE}%{WORD:msg_level}%{SPACE}:%{SPACE}%{JAVACLASS:javaclass}%{SPACE}:%{SPACE}%{GREEDYDATA:narrative}" }
                            remove_tag => ["_grokparsefailure"]
                            add_tag => [ "grok2707" ]
                            add_field => { "my_body" => "%{msg_level} from %{host}:%{path}:\n\n%{narrative}\n\nJavaClass: %{javaclass}" }
                    }
            }

            if( "_grokparsefailure" in [tags] ) {
                    mutate { add_tag => [ "grok2799" ] }
            }
    }
7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.