File input breaks my logstash config

Elastic Stack Logstash
1

I was previously using the stdin input plugin for testing and while startup was slow, it worked. When I switched to the file input startup takes ages and never actually completes (I don't get the startup completed message, and the cpu util drops to near 0 after about an hour).

My config is below, and I run logstash via /opt/logstash/bin/logstash -f /etc/logstash/conf.d/testing.conf | tee :

input {
  file {
    path => [
         "/tmp/logstash/scannet70/ag_log",
         "/tmp/logstash/scannet78/ag_log",
         "/tmp/logstash/scannet86/ag_log",
         "/tmp/logstash/scannet89/ag_log",
         "/tmp/logstash/scannet93/ag_log",
         "/tmp/logstash/scannet94/ag_log"
            ]
    stat_interval => 60
    start_position => "beginning"
  }
}

filter {
  multiline {
    pattern => "^(\[|started|Resetting)"
    negate => true
    what => "previous"
  }
  grok {
    patterns_dir => "/etc/logstash/conf.d/patterns"
    match => {"message" => ".*"}
    match => { "message" => [
                             "started at %{NUMBER:timestamp}, local %{TIMESTAMP_ISO8601:localtime}",
                             "Resetting the coordinator, waiting 30 seconds\.\.\.%{RESET_MULTILINE:line}",
                             "\[%{TIME_FORMAT:timestamp}: %{NONNEGINT:device}\] %{DEVICE_MESSAGE:line}",
                             "\[%{TIME_FORMAT:timestamp}: %{MOTE:mote}\] %{MOTE_MESSAGE:line}",
                             "\[%{TIME_FORMAT:timestamp}: %{MOTE:mote}\] %{CHILD_MESSAGE:line}",
                             "\[%{TIME_FORMAT:timestamp}: (?<command>mote command)\] %{MOTE_COMMAND:line}",
                             "\[%{TIME_FORMAT:timestamp}: (?<command>aggregation)\] entered isInNwkSession %{NONNEGINT:session}",
                             "\[%{TIME_FORMAT:timestamp}: (?<command>network)\] %{NETWORK_MESSAGE:line}%{NETWORK_MULTILINE:multiline}",
                             "\[%{TIME_FORMAT:timestamp}: (?<command>ms cmd)\] %{MS_CMD:line}",
                             "\[%{TIME_FORMAT:timestamp}: (?<command>aggregator)\]%{AGGREGATOR_MESSAGE:line}",
                             "\[%{TIME_FORMAT:timestamp}: (?<command>aggregator)\] %{AGGREGATOR_MESSAGE_TYPE_2:line}"
                            ] }
  }
  if ([timestamp] !~ /.+/) {
      drop {}
    }
    mutate {
      convert => { "acc_rate" => "float" }
      convert => { "address" => "integer" }
      convert => { "attempt" => "integer" }
      # many more converts removed in order to fit here
      add_field => { "gateway" => "%{path}" }
      gsub =>  [ "gateway", ".*scannet([0-9]+).*", ""]
      gsub =>  [ "timestamp", "(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)", "2015 \1"]
    }
    translate {
      field => "gateway"
      destination => "project"
      dictionary => {
        "70" => "P247"
        "78" => "P237"
        "86" => "P269"
        "89" => "P263"
        "93" => "P272"
        "94" => "P273"
      }   
    }
    date {
      match => [ "timestamp", "yyyy MMM dd, HH:mm:ss", "UNIX"]
    }
  }

  output {
    stdout {
      codec => rubydebug
    }
  }
2

Hi,

Can you run logstash from the command line with --debug option and see where it stops.

3

After a huge number of the following line the process hangs and cpu use goes to ~0.

{:timestamp=>"2016-01-12T15:32:20.499000+0000", :message=>"Flushing", :plugin=><LogStash::Filters::Multiline pattern=>"^(\\[|started|Resetting)", negate=>true, what=>"previous", periodic_flush=>true, source=>"message", allow_duplicates=>true, stream_identity=>"%{host}.%{path}.%{type}", max_age=>5>, :level=>:debug, :file=>"(eval)", :line=>"16", :method=>"initialize"}

4

I tried to simplify my case and made a new topic here