Bug on kv remove_char_value => "\\"

[junnester]

• Version: logstash-6.2.4 -- filter plugin kv

• Operating System:

  • CentOS Linux release 7.4.1708 (Core)
  • Linux 3.10.0-693.2.2.el7.x86_64 #1 SMP Tue Sep 12 22:26:13 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux

• Config File :

input {
  file {
    path => "/var/log/monitoring/mimecast/mimecast_siem.*.log"
    type => "mimecast_Production"
  }
}


filter {
  if [type] == "mimecast_Production" {
    # messages are pipe '|' delimited
    kv { 
      field_split => "\|"
      # BUG: cannot remove solitary char backslash '\'
      remove_char_value => "\\" 
    }
    mutate { remove_field => [ "message" ] }
    date { match => [ "datetime", "ISO8601"] }

    # is a Reject message
    if "RejType" { mutate { add_tag => "mimecast_reject" } }

    # Message directions: sent, recieved, internal
    if "Outbound" in [Dir] {
      mutate { add_tag => "mimecast_recieved" }
    } else if "Inbound" in [Dir] {
      mutate { add_tag => "mimecast_sent" }
    } else if "Internal" in [Dir] {
      mutate { add_tag => "mimecast_internal" }
    }
    
    # is Spam messag
    if [SpamInfo] { mutate { add_tag => "mimecast_spam"} }
  }
}


output {
  if [type] == "mimecast_Production" {
    stdout { codec => rubydebug }
    # send message graphite
    # TODO
    # sendtographite {} ??
  } 
}
 

• Sample Data:

datetime=2017-05-26T16:47:41+0100|aCode=7O7I7MvGP1mj8plHRDuHEA|acc=C0A0|SpamLimit=0|IP=123.123.123.123|Dir=Internal|MsgId=<messageId@messageId>|Subject=\message subject\|headerFrom=from@mimecast.com|Sender=from@mimecast.com|Rcpt=auser@mimecast.com|SpamInfo=[]|Act=Acc|TlsVer=TLSv1|Cphr=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA|SpamScore=1

• Steps to Reproduce: Run with filter - kv and try to remove only backslash \
  this causes config errors. see 'Example Logstash ERROR below'

filter {
  kv {
    remove_char_value => "\\"
  }
}

• Example Logstash ERROR

[2018-05-03T12:17:08,940][ERROR][logstash.agent           ] Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Expected one of #, {, } at line 18, column 33 (byte 401) after filter {\n  if [type] == \"mimecast_Production\" {\n    # messages are pipe '|' delimited\n    kv { \n      field_split => \"\\|\"\n      # BUG: cannot remove solitary char backslash '\\', adding comma ',' as banaide\n      remove_char_value => \"\\\\\" \n    }\n    mutate { remove_field => [ \"", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:42:in `compile_imperative'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:50:in `compile_graph'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:12:in `block in compile_sources'", "org/jruby/RubyArray.java:2486:in `map'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:11:in `compile_sources'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:51:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/reload.rb:34:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:315:in `block in converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:312:in `block in converge_state'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:299:in `converge_state'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:166:in `block in converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:141:in `with_pipelines'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:164:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:105:in `block in execute'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/interval.rb:18:in `interval'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:94:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:348:in `block in execute'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'"]}