Logstash Sending older logs

Elastic Stack Logstash
1

Logstash is sending older messages. Below is my pipeline.

input {
  file {
    path => "C:/Program Files (x86)/dataops/ag/web/Logs/webSvr*"
    type => "webSvr"
    start_position => "beginning"
  }
  file {
    path => "C:/Program Files (x86)/dataops/webserver/Server/logs/webserver.log"
    type => "webserver.log"
    start_position => "beginning"
  }
  file {
    path => "C:/Program Files (x86)/dataops/webserver/Server/logs/app.log"
    type => "app"
    codec => multiline {
      pattern => "^%{TIMESTAMP_ISO8601}"
      negate => true
      what => "previous"
      charset => "ISO-8859-1"
    }
    start_position => "beginning"
  }
}

filter {
  if [type] == "webSvr" {
    grok {
      match => {"message" => "^%{INT:logtimestamp}%{GREEDYDATA:message}"}
      overwrite => [ "message"]
    }
    mutate {
      remove_field => [ "logtimeStamp" ]
    }
  }
  if [type] == "webserver.log" {
    grok {
      match => {"message" => "\[%{HTTPDATE:logtimeStamp}\] %{IP:hostip} %{URIPROTO:method} %{URIPATH:post-data} (?:%{NOTSPACE:queryparam}|-) %{NUMBER:useragent} %{NUMBER:responsestatus} \[%{GREEDYDATA:message}\] - %{NUMBER:time-taken:int}"}
      overwrite => [ "message"]
    }
    mutate {
      remove_field => [ "logtimeStamp" ]
    }
  }
  if [type] == "app" {
    mutate {
      gsub => [
        "message", "\[\] ", " ",
        "message", "\- ", " ",
        "message", "\s+", " "
      ]
    }
    mutate {
      strip => ["message"]
    }
    grok {
      match => {"message" => ["%{TIMESTAMP_ISO8601:logtimeStamp} %{WORD:loglevel} \[%{USERNAME:httpcall}] %{USERNAME:dbName} %{USERNAME:tenant} %{INT:tenantId} %{INT:userId} %{USERNAME:session} %{GREEDYDATA:message}",
                              "%{TIMESTAMP_ISO8601:logtimeStamp} %{WORD:loglevel} %{GREEDYDATA:message}" ]}
      overwrite => [ "message" ]
    }
    mutate {
      remove_field => [ "logtimeStamp" ]
    }
  }
}

filter {
  fingerprint {
    method => "SHA1"
    key => "103013"
  }
}


output {
  if [type] == "web" {
    elasticsearch {
      ecs_compatibility => disabled
      hosts => "***************************:443"
      ssl => true
      index => "${NODE_ROLE}-traffic-%{+YYYY.MM.dd}"

    }
  }
  else {
    elasticsearch {
      ecs_compatibility => disabled
      hosts => "*****************:443"
      ssl => true
      index => "${NODE_ROLE}-%{+YYYY.MM.dd}"

    }
  }
}

please suggest.

2

What makes you think that is an issue?

3

In log file new events are present.

In elk I see August 13 messages instead on August 16th isn’t this issue?

4

No, the default behavior of the file input when you configure it to read a path is to read every file in that directory.

You can however use the ignore_older option to configure it to ignore files that have a modified date older than some specific time in seconds.

5

Also, you set start_position => "beginning", so if a log file covers multiple days you have asked for all the old data to be included. Perhaps start_position => "end" would work better for you.

2 Likes
6

thank you @Badger and @leandrojmp. Just a thought after looking at the documentation. In case if I have a fingerprint filter. Will this sort the issue.